You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/06/20 18:13:38 UTC
svn commit: r1352199 - in /incubator/oozie/trunk: ./ core/
core/src/main/conf/ core/src/test/java/org/apache/oozie/action/hadoop/
hadooplibs/ hadooplibs/hadoop-distcp-1/ hadooplibs/hadoop-distcp-2/
sharelib/ sharelib/distcp/ src/main/assemblies/
Author: tucu
Date: Wed Jun 20 16:13:37 2012
New Revision: 1352199
URL: http://svn.apache.org/viewvc?rev=1352199&view=rev
Log:
OOZIE-766 Verify distcp action works with Hadoop 0.23 (rkanter via tucu)
Added:
incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/ (with props)
incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/pom.xml
incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/ (with props)
incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/pom.xml
incubator/oozie/trunk/sharelib/distcp/ (with props)
incubator/oozie/trunk/sharelib/distcp/pom.xml
Modified:
incubator/oozie/trunk/core/ (props changed)
incubator/oozie/trunk/core/pom.xml
incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
incubator/oozie/trunk/hadooplibs/pom.xml
incubator/oozie/trunk/pom.xml
incubator/oozie/trunk/release-log.txt
incubator/oozie/trunk/sharelib/pom.xml
incubator/oozie/trunk/src/main/assemblies/sharelib.xml
Propchange: incubator/oozie/trunk/core/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Wed Jun 20 16:13:37 2012
@@ -1 +1,2 @@
target
+build
Modified: incubator/oozie/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/pom.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/core/pom.xml (original)
+++ incubator/oozie/trunk/core/pom.xml Wed Jun 20 16:13:37 2012
@@ -222,6 +222,12 @@
<dependency>
<groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-distcp</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
<artifactId>oozie-sharelib-hive</artifactId>
<scope>provided</scope>
</dependency>
Modified: incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/conf/oozie-site.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/conf/oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/main/conf/oozie-site.xml Wed Jun 20 16:13:37 2012
@@ -29,13 +29,14 @@
org.apache.oozie.action.email.EmailActionExecutor,
org.apache.oozie.action.hadoop.HiveActionExecutor,
org.apache.oozie.action.hadoop.ShellActionExecutor,
- org.apache.oozie.action.hadoop.SqoopActionExecutor
+ org.apache.oozie.action.hadoop.SqoopActionExecutor,
+ org.apache.oozie.action.hadoop.DistcpActionExecutor
</value>
</property>
<property>
<name>oozie.service.SchemaService.wf.ext.schemas</name>
- <value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd,ssh-action-0.1.xsd</value>
+ <value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd,ssh-action-0.1.xsd,distcp-action-0.1.xsd</value>
</property>
<property>
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java Wed Jun 20 16:13:37 2012
@@ -44,13 +44,20 @@ public class TestDistCpActionExecutor ex
setSystemProperty("oozie.service.ActionService.executor.classes", DistcpActionExecutor.class.getName());
}
- public void testSimpestdistCpSubmitOK() throws Exception {
+ public void testDistCpFile() throws Exception {
+ Path inputPath = new Path(getFsTestCaseDir(), "input.txt");
+ final Path outputPath = new Path(getFsTestCaseDir(), "output.txt");
+ byte[] content = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes();
+
+ OutputStream os = getFileSystem().create(inputPath);
+ os.write(content);
+ os.close();
+
String actionXml = "<distcp>" +
"<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>" +
- "<arg>-Dmyqueue</arg>" +
- "<arg>input</arg>"+
- "<arg>output</arg>" +
+ "<arg>" + inputPath + "</arg>"+
+ "<arg>" + outputPath + "</arg>" +
"</distcp>";
Context context = createContext(actionXml);
final RunningJob runningJob = submitAction(context);
@@ -60,6 +67,33 @@ public class TestDistCpActionExecutor ex
}
});
assertTrue(runningJob.isSuccessful());
+
+ waitFor(60 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return getFileSystem().exists(outputPath);
+ }
+ });
+ assertTrue(getFileSystem().exists(outputPath));
+
+ byte[] readContent = new byte[content.length];
+ InputStream is = getFileSystem().open(outputPath);
+ int offset = 0;
+ while (offset < readContent.length)
+ {
+ int numRead = is.read(readContent, offset, readContent.length);
+ if(numRead == -1) {
+ break;
+ }
+ offset += numRead;
+ }
+ assertEquals(is.read(), -1);
+ is.close();
+ offset = 0;
+ while (offset < readContent.length)
+ {
+ assertEquals(readContent[offset], content[offset]);
+ offset++;
+ }
}
Propchange: incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Jun 20 16:13:37 2012
@@ -0,0 +1 @@
+target
Added: incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/pom.xml?rev=1352199&view=auto
==============================================================================
--- incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/pom.xml (added)
+++ incubator/oozie/trunk/hadooplibs/hadoop-distcp-1/pom.xml Wed Jun 20 16:13:37 2012
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>3.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-distcp</artifactId>
+ <version>1.0.1.oozie-3.3.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Distcp ${project.version}</description>
+ <name>Apache Oozie Hadoop Distcp ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-tools</artifactId>
+ <version>1.0.1</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+</project>
+
Propchange: incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Jun 20 16:13:37 2012
@@ -0,0 +1 @@
+target
Added: incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/pom.xml?rev=1352199&view=auto
==============================================================================
--- incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/pom.xml (added)
+++ incubator/oozie/trunk/hadooplibs/hadoop-distcp-2/pom.xml Wed Jun 20 16:13:37 2012
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>3.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-distcp</artifactId>
+ <version>2.0.0-SNAPSHOT.oozie-3.3.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Distcp ${project.version}</description>
+ <name>Apache Oozie Hadoop Distcp ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+</project>
+
Modified: incubator/oozie/trunk/hadooplibs/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/hadooplibs/pom.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/hadooplibs/pom.xml (original)
+++ incubator/oozie/trunk/hadooplibs/pom.xml Wed Jun 20 16:13:37 2012
@@ -33,9 +33,11 @@
<modules>
<module>hadoop-1</module>
+ <module>hadoop-distcp-1</module>
<module>hadoop-test-1</module>
<module>hadoop-2</module>
<module>hadoop-test-2</module>
+ <module>hadoop-distcp-2</module>
</modules>
<build>
Modified: incubator/oozie/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/pom.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/pom.xml (original)
+++ incubator/oozie/trunk/pom.xml Wed Jun 20 16:13:37 2012
@@ -82,6 +82,7 @@
<pig.version>0.9.0</pig.version>
<sqoop.version>1.5.0-incubating-SNAPSHOT</sqoop.version>
<streaming.version>${hadoop.version}</streaming.version>
+ <distcp.version>${hadooplib.version}</distcp.version>
</properties>
<modules>
@@ -208,6 +209,11 @@
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-distcp</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
<artifactId>oozie-docs</artifactId>
<version>${project.version}</version>
<type>war</type>
@@ -312,6 +318,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-distcp</artifactId>
+ <version>${distcp.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Wed Jun 20 16:13:37 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-766 Verify distcp action works with Hadoop 0.23 (rkanter via tucu)
OOZIE-801 Accepts coordinator with start time > end time (svenkat via tucu)
OOZIE-867 Unit test to account for log retrieval from multiple gzipped oozie.log files (mona via tucu)
OOZIE-649 Fail fast when a date doesn't parse correctly (rkanter via tucu)
Propchange: incubator/oozie/trunk/sharelib/distcp/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Jun 20 16:13:37 2012
@@ -0,0 +1 @@
+target
Added: incubator/oozie/trunk/sharelib/distcp/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/distcp/pom.xml?rev=1352199&view=auto
==============================================================================
--- incubator/oozie/trunk/sharelib/distcp/pom.xml (added)
+++ incubator/oozie/trunk/sharelib/distcp/pom.xml Wed Jun 20 16:13:37 2012
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>3.3.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-distcp</artifactId>
+ <version>3.3.0-SNAPSHOT</version>
+ <description>Apache Oozie Share Lib Distcp</description>
+ <name>Apache Oozie Share Lib Distcp</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <sharelib.action.postfix>distcp</sharelib.action.postfix>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-distcp</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <finalName>partial-sharelib</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>../../src/main/assemblies/partial-sharelib.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
Modified: incubator/oozie/trunk/sharelib/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/pom.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/pom.xml Wed Jun 20 16:13:37 2012
@@ -37,6 +37,7 @@
<module>hive</module>
<module>sqoop</module>
<module>oozie</module>
+ <module>distcp</module>
</modules>
<build>
Modified: incubator/oozie/trunk/src/main/assemblies/sharelib.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/src/main/assemblies/sharelib.xml?rev=1352199&r1=1352198&r2=1352199&view=diff
==============================================================================
--- incubator/oozie/trunk/src/main/assemblies/sharelib.xml (original)
+++ incubator/oozie/trunk/src/main/assemblies/sharelib.xml Wed Jun 20 16:13:37 2012
@@ -51,6 +51,10 @@
<directory>${basedir}/oozie/target/partial-sharelib</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/distcp/target/partial-sharelib</directory>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
</fileSets>
</assembly>