You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/02/25 22:42:09 UTC
svn commit: r1449911 [5/8] - in /oozie/trunk: ./
client/src/main/java/org/apache/oozie/cli/
client/src/main/java/org/apache/oozie/client/
client/src/main/java/org/apache/oozie/client/rest/
client/src/test/java/org/apache/oozie/client/rest/ core/ core/s...
Modified: oozie/trunk/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ oozie/trunk/core/src/main/resources/oozie-default.xml Mon Feb 25 21:42:07 2013
@@ -8,9 +8,9 @@
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -90,7 +90,9 @@
org.apache.oozie.service.UUIDService,
org.apache.oozie.service.ELService,
org.apache.oozie.service.AuthorizationService,
+ org.apache.oozie.service.UserGroupInformationService,
org.apache.oozie.service.HadoopAccessorService,
+ org.apache.oozie.service.URIHandlerService,
org.apache.oozie.service.MemoryLocksService,
org.apache.oozie.service.DagXLogInfoService,
org.apache.oozie.service.SchemaService,
@@ -129,6 +131,34 @@
</description>
</property>
+ <!-- HCatAccessorService -->
+ <property>
+ <name>oozie.service.HCatAccessorService.connections</name>
+ <value>
+ default=java.naming.factory.initial#org.apache.activemq.jndi.ActiveMQInitialContextFactory;java.naming.provider.url#tcp://localhost:61616;connectionFactoryNames#ConnectionFactory
+ </value>
+ <description>
+ Specify the map of endpoints to JMS configuration properties. In general, endpoint
+ identifies the HCatalog server URL. "default" is used if no endpoint is mentioned
+ in the query. If some JMS property is not defined, the system will use the property
+ defined jndi.properties. jndi.properties files is retrieved from the application classpath.
+ Mapping rules can also be provided for mapping Hcatalog server names to JMS server.
+ hcat://${1}.${2}.server.com:8020=java.naming.factory.initial#Dummy.Factory;java.naming.provider.url#tcp://broker.${2}:61616
+ </description>
+ </property>
+
+ <!-- JMSAccessorService -->
+ <property>
+ <name>oozie.service.JMSAccessorService.connectioncontext.impl</name>
+ <value>
+ org.apache.oozie.jms.DefaultConnectionContext
+ </value>
+ <description>
+ Specifies the Connection Context implementation
+ </description>
+ </property>
+
+
<!-- ConfigurationService -->
<property>
@@ -244,6 +274,15 @@
</property>
<property>
+ <name>oozie.service.RecoveryService.push.dependency.interval</name>
+ <value>200</value>
+ <description>
+ This value determines the delay for push missing dependency command queueing
+ in Recovery Service
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.RecoveryService.interval</name>
<value>60</value>
<description>
@@ -377,7 +416,15 @@
</description>
</property>
- <property>
+ <property>
+ <name>oozie.service.coord.push.check.requeue.interval
+ </name>
+ <value>600000</value>
+ <description>Command re-queue interval for push dependencies (in millisecond).
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.coord.default.concurrency
</name>
<value>1</value>
@@ -492,7 +539,8 @@
fs:isDir=org.apache.oozie.action.hadoop.FsELFunctions#fs_isDir,
fs:dirSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_dirSize,
fs:fileSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_fileSize,
- fs:blockSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_blockSize
+ fs:blockSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_blockSize,
+ hcat:exists=org.apache.oozie.coord.HCatELFunctions#hcat_exists
</value>
<description>
EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
@@ -740,7 +788,16 @@
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo,
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
- coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
+ coord:databaseIn=org.apache.oozie.coord.HCatELFunctions#ph1_coord_databaseIn_echo,
+ coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_databaseOut_echo,
+ coord:tableIn=org.apache.oozie.coord.HCatELFunctions#ph1_coord_tableIn_echo,
+ coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_tableOut_echo,
+ coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionFilter_echo,
+ coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMin_echo,
+ coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMax_echo,
+ coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitions_echo,
+ coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo
</value>
<description>
EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
@@ -993,7 +1050,16 @@
coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actionId,
coord:name=org.apache.oozie.coord.CoordELFunctions#ph3_coord_name,
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
- coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
+ coord:databaseIn=org.apache.oozie.coord.HCatELFunctions#ph3_coord_databaseIn,
+ coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_databaseOut,
+ coord:tableIn=org.apache.oozie.coord.HCatELFunctions#ph3_coord_tableIn,
+ coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_tableOut,
+ coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionFilter,
+ coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMin,
+ coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMax,
+ coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitions,
+ coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue
</value>
<description>
EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
@@ -1680,6 +1746,13 @@
</description>
</property>
+ <property>
+ <name>oozie.service.URIHandlerService.uri.handlers</name>
+ <value>org.apache.oozie.dependency.FSURIHandler</value>
+ <description>
+ Enlist the different uri schemes supported for data availability checks.
+ </description>
+ </property>
<!-- Oozie HTTP Notifications -->
<property>
Modified: oozie/trunk/core/src/main/resources/oozie-log4j.properties
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/resources/oozie-log4j.properties?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/main/resources/oozie-log4j.properties (original)
+++ oozie/trunk/core/src/main/resources/oozie-log4j.properties Mon Feb 25 21:42:07 2013
@@ -40,6 +40,7 @@ log4j.logger.oozieaudit=NONE, none
log4j.logger.org.apache.oozie.local=DEBUG, test
log4j.logger.org.apache.oozie.client=DEBUG, test
log4j.logger.org.apache.oozie.test=DEBUG, test
+log4j.logger.org.apache.oozie.dependency=DEBUG, test
log4j.logger.org.apache.oozie.wf=DEBUG, test
log4j.logger.org.apache.oozie.action=DEBUG, test
log4j.logger.org.apache.oozie.command=DEBUG, test
Added: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFSPrepareActions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFSPrepareActions.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFSPrepareActions.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFSPrepareActions.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+
+public class TestFSPrepareActions extends XFsTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ Services services = new Services();
+ services.getConf().set(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "hdfs");
+ services.init();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ // Test for delete as prepare action
+ @Test
+ public void testDelete() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+ fs.mkdirs(newDir);
+ // Prepare block that contains delete action
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
+
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
+ assertFalse(fs.exists(newDir));
+ }
+
+ // Test for mkdir as prepare action
+ @Test
+ public void testMkdir() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+ // Prepare block that contains mkdir action
+ String prepareXML = "<prepare>" + "<mkdir path='" + newDir + "'/>" + "</prepare>";
+
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
+ assertTrue(fs.exists(newDir));
+ }
+
+ // Test for invalid scheme value in the path for action
+ @Test
+ public void testForInvalidScheme() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ // Construct a path with invalid scheme
+ Path newDir = new Path("hftp:/" + actionDir.toString().substring(5) + "/delete");
+ // Construct prepare XML block with the path
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ try {
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
+ fail("Expected to catch an exception but did not encounter any");
+ } catch (LauncherException le) {
+ Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
+ assertEquals("Scheme hftp not supported in uri " + path, le.getMessage());
+ } catch(Exception ex){
+ fail("Expected a LauncherException but received an Exception");
+ }
+ }
+
+ // Test for null scheme value in the path for action
+ @Test
+ public void testForNullScheme() throws Exception {
+ // Construct a path without scheme
+ Path newDir = new Path("test/oozietests/testDelete/delete");
+ FileSystem fs = getFileSystem();
+ // Delete the file if it is already there
+ if (fs.exists(newDir)) {
+ fs.delete(newDir, true);
+ }
+ // Construct prepare XML block with the path
+ String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
+
+ // Parse the XML to get the node
+ Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
+ Node n = doc.getDocumentElement().getChildNodes().item(0);
+
+ try {
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
+ fail("Expected to catch an exception but did not encounter any");
+ } catch (LauncherException le) {
+ Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
+ assertEquals("Scheme not present in uri " + path, le.getMessage());
+ } catch(Exception ex) {
+ fail("Expected a LauncherException but received an Exception");
+ }
+ }
+
+}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java Mon Feb 25 21:42:07 2013
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.oozie.service.HadoopAccessorService;
-import org.apache.oozie.test.XFsTestCase;
-import org.apache.oozie.service.Services;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.Arrays;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-public class TestFileSystemActions extends XFsTestCase {
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- new Services().init();
- }
-
- @Override
- protected void tearDown() throws Exception {
- Services.get().destroy();
- super.tearDown();
- }
-
- // Test for delete as prepare action
- public void testDelete() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- Path newDir = new Path(actionDir, "newDir");
- // Delete the file if it is already there
- if (fs.exists(newDir)) {
- fs.delete(newDir, true);
- }
- fs.mkdirs(newDir);
- // Prepare block that contains delete action
- String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
-
- // Parse the XML to get the node
- Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
- Node n = doc.getDocumentElement().getChildNodes().item(0);
-
- new FileSystemActions(Arrays.asList("hdfs")).execute(n);
- assertFalse(fs.exists(newDir));
- }
-
- // Test for mkdir as prepare action
- public void testMkdir() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- Path newDir = new Path(actionDir, "newDir");
- // Delete the file if it is already there
- if (fs.exists(newDir)) {
- fs.delete(newDir, true);
- }
-
- // Prepare block that contains mkdir action
- String prepareXML = "<prepare>" + "<mkdir path='" + newDir + "'/>" + "</prepare>";
-
- // Parse the XML to get the node
- Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
- Node n = doc.getDocumentElement().getChildNodes().item(0);
-
- new FileSystemActions(Arrays.asList("hdfs")).execute(n);
- assertTrue(fs.exists(newDir));
- }
-
- // Test for invalid scheme value in the path for action
- public void testForInvalidScheme() throws Exception {
- Path actionDir = getFsTestCaseDir();
- // Construct a path with invalid scheme
- Path newDir = new Path("file:/" + actionDir.toString().substring(5) + "/delete");
- // Construct prepare XML block with the path
- String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
- // Parse the XML to get the node
- Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
- Node n = doc.getDocumentElement().getChildNodes().item(0);
-
- try {
- new FileSystemActions(Arrays.asList("hdfs")).execute(n);
- fail("Expected to catch an exception but did not encounter any");
- } catch (LauncherException le) {
- Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
- assertEquals("Scheme of '" + path + "' is not supported.", le.getMessage());
- } catch(Exception ex){
- fail("Expected a LauncherException but received an Exception");
- }
- }
-
- // Test for null scheme value in the path for action
- public void testForNullScheme() throws Exception {
- // Construct a path without scheme
- Path newDir = new Path("test/oozietests/testDelete/delete");
- FileSystem fs = getFileSystem();
- // Delete the file if it is already there
- if (fs.exists(newDir)) {
- fs.delete(newDir, true);
- }
- // Construct prepare XML block with the path
- String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
-
- // Parse the XML to get the node
- Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
- Node n = doc.getDocumentElement().getChildNodes().item(0);
-
- try {
- new FileSystemActions(Arrays.asList("hdfs")).execute(n);
- fail("Expected to catch an exception but did not encounter any");
- } catch (LauncherException le) {
- Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
- assertEquals("Scheme of the path " + path + " is null", le.getMessage());
- } catch(Exception ex) {
- fail("Expected a LauncherException but received an Exception");
- }
- }
-
-}
Added: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatPrepareActions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatPrepareActions.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatPrepareActions.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatPrepareActions.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.oozie.test.XHCatTestCase;
+import org.apache.oozie.dependency.FSURIHandler;
+import org.apache.oozie.dependency.HCatURIHandler;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
+import org.junit.Test;
+
+public class TestHCatPrepareActions extends XHCatTestCase {
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ Services services = new Services();
+ services.getConf().set(URIHandlerService.URI_HANDLERS,
+ FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
+ services.init();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ dropTable("db1", "table1", true);
+ dropDatabase("db1", true);
+ createDatabase("db1");
+ createTable("db1", "table1", "year,month,dt,country");
+ String part1 = addPartition("db1", "table1", "year=2012;month=12;dt=02;country=us");
+ String part2 = addPartition("db1", "table1", "year=2012;month=12;dt=03;country=us");
+ String part3 = addPartition("db1", "table1", "year=2013;month=1;dt=01;country=us");
+
+ String uri1 = "hcat://" + getMetastoreAuthority() + "/db1/table1/year=2012;month=12";
+ String uri2 = "hcat://" + getMetastoreAuthority() + "/db1/table1/year=2013;dt=01";
+
+ // Prepare block that contains delete action
+ String prepareXML = "<prepare>"
+ + "<delete path='" + uri1 + "'/>"
+ + "<delete path='" + uri2 + "'/>"
+ + "</prepare>";
+
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
+ FileSystem fs = getFileSystem();
+ assertFalse(fs.exists(new Path(part1)));
+ assertFalse(fs.exists(new Path(part2)));
+ assertFalse(fs.exists(new Path(part3)));
+ assertEquals(0, getPartitions("db1", "table1", "country=us").size());
+ dropTable("db1", "table1", true);
+ dropDatabase("db1", true);
+ }
+
+}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java Mon Feb 25 21:42:07 2013
@@ -19,6 +19,7 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.MiniHCatServer;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import java.io.File;
@@ -122,6 +123,8 @@ public class TestHiveMain extends MainTe
os = new FileOutputStream(hiveSite);
jobConf.writeXml(os);
os.close();
+ MiniHCatServer.resetDefaultDBCreation();
+ MiniHCatServer.resetHiveConfStaticVariables();
HiveMain.main(null);
}
catch (SecurityException ex) {
@@ -139,6 +142,7 @@ public class TestHiveMain extends MainTe
finally {
System.setProperty("user.name", user);
hiveSite.delete();
+ MiniHCatServer.resetHiveConfStaticVariables();
}
assertTrue(outputDataFile.exists());
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java Mon Feb 25 21:42:07 2013
@@ -46,13 +46,13 @@ import org.apache.oozie.WorkflowActionBe
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.ActionExecutorException;
-import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
@@ -66,6 +66,8 @@ import org.apache.oozie.workflow.lite.En
import org.apache.oozie.workflow.lite.LiteWorkflowApp;
import org.apache.oozie.workflow.lite.StartNodeDef;
import org.jdom.Element;
+import org.junit.Assert;
+import org.junit.Test;
public class TestJavaActionExecutor extends ActionExecutorTestCase {
@@ -75,6 +77,7 @@ public class TestJavaActionExecutor exte
setSystemProperty("oozie.service.ActionService.executor.classes", JavaActionExecutor.class.getName());
setSystemProperty("oozie.service.HadoopAccessorService.action.configurations",
"*=hadoop-conf," + getJobTrackerUri() + "=action-conf");
+ setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH, getFsTestCaseDir() + "/systemlib");
new File(getTestCaseConfDir(), "action-conf").mkdir();
InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("test-action-config.xml");
OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() + "/action-conf", "java.xml"));
@@ -96,8 +99,8 @@ public class TestJavaActionExecutor exte
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
- classes.add(FileSystemActions.class);
classes.add(PrepareActionsDriver.class);
+ classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
classes.add(ActionStats.class);
classes.add(ActionType.class);
assertEquals(classes, ae.getLauncherClasses());
@@ -183,7 +186,7 @@ public class TestJavaActionExecutor exte
OutputStream os = getFileSystem().create(new Path(getFsTestCaseDir(), "job.xml"));
conf.writeXml(os);
os.close();
-
+
conf = new XConfiguration();
conf.set("e", "E");
conf.set("oozie.launcher.f", "F");
@@ -260,7 +263,6 @@ public class TestJavaActionExecutor exte
assertEquals("MAIN-CLASS", ae.getLauncherMain(conf, actionXml));
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS"));
assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf)));
- assertNotNull(conf.get(LauncherMapper.CONF_OOZIE_ACTION_SUPPORTED_FILESYSTEMS));
assertTrue(getFileSystem().exists(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)));
@@ -318,7 +320,6 @@ public class TestJavaActionExecutor exte
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, appJarPath.toString(), appSoPath.toString());
-
WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
if(group != null) {
wf.setGroup(group);
@@ -983,20 +984,24 @@ public class TestJavaActionExecutor exte
Configuration actionConf = new XConfiguration();
- assertEquals("java-action-executor", ae.getShareLibName(context, new Element("java"), actionConf));
+ Assert.assertArrayEquals(new String[] { "java-action-executor" },
+ ae.getShareLibNames(context, new Element("java"), actionConf));
Services.get().getConf().set("oozie.action.sharelib.for.java", "java-oozie-conf");
- assertEquals("java-oozie-conf", ae.getShareLibName(context, new Element("java"), actionConf));
+ Assert.assertArrayEquals(new String[] { "java-oozie-conf" },
+ ae.getShareLibNames(context, new Element("java"), actionConf));
jobConf = "<configuration>" + "<property>"
+ "<name>oozie.action.sharelib.for.java</name>"
+ "<value>java-job-conf</value>" + "</property>"
+ "</configuration>";
wfBean.setConf(jobConf);
- assertEquals("java-job-conf", ae.getShareLibName(context, new Element("java"), actionConf));
+ Assert.assertArrayEquals(new String[] { "java-job-conf" },
+ ae.getShareLibNames(context, new Element("java"), actionConf));
actionConf.set("oozie.action.sharelib.for.java", "java-action-conf");
- assertEquals("java-action-conf", ae.getShareLibName(context, new Element("java"), actionConf));
+ Assert.assertArrayEquals(new String[] { "java-action-conf" },
+ ae.getShareLibNames(context, new Element("java"), actionConf));
}
public void testJavaOpts() throws Exception {
@@ -1051,76 +1056,76 @@ public class TestJavaActionExecutor exte
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPT1"));
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPT2"));
}
-
+
public void testActionLibsPath() throws Exception {
// Test adding a directory
Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs");
- getFileSystem().mkdirs(actionLibPath);
+ getFileSystem().mkdirs(actionLibPath);
Path jar1Path = new Path(actionLibPath, "jar1.jar");
getFileSystem().create(jar1Path).close();
Path jar2Path = new Path(actionLibPath, "jar2.jar");
getFileSystem().create(jar2Path).close();
-
+
String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>" +
"<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "</value></property>" +
+ "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "</value></property>" +
"</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
"</java>";
Element eActionXml = XmlUtils.parseXml(actionXml);
Context context = createContext(actionXml, null);
-
+
JavaActionExecutor ae = new JavaActionExecutor();
Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
+
URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
String cacheFilesStr = Arrays.toString(cacheFiles);
assertTrue(cacheFilesStr.contains(jar1Path.toString()));
assertTrue(cacheFilesStr.contains(jar2Path.toString()));
-
+
// Test adding a file
Path jar3Path = new Path(getFsTestCaseDir(), "jar3.jar");
getFileSystem().create(jar3Path).close();
-
+
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>" +
"<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" + jar3Path + "</value></property>" +
+ "<property><name>oozie.launcher.oozie.libpath</name><value>" + jar3Path + "</value></property>" +
"</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
"</java>";
eActionXml = XmlUtils.parseXml(actionXml);
context = createContext(actionXml, null);
-
+
ae = new JavaActionExecutor();
jobConf = ae.createBaseHadoopConf(context, eActionXml);
ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
+
cacheFiles = DistributedCache.getCacheFiles(jobConf);
cacheFilesStr = Arrays.toString(cacheFiles);
assertTrue(cacheFilesStr.contains(jar3Path.toString()));
-
+
// Test adding a directory and a file (comma separated)
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>" +
"<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" +
- "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "," + jar3Path +
- "</value></property>" +
+ "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "," + jar3Path +
+ "</value></property>" +
"</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
"</java>";
eActionXml = XmlUtils.parseXml(actionXml);
context = createContext(actionXml, null);
-
+
ae = new JavaActionExecutor();
jobConf = ae.createBaseHadoopConf(context, eActionXml);
ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
-
+
cacheFiles = DistributedCache.getCacheFiles(jobConf);
cacheFilesStr = Arrays.toString(cacheFiles);
assertTrue(cacheFilesStr.contains(jar1Path.toString()));
@@ -1128,6 +1133,85 @@ public class TestJavaActionExecutor exte
assertTrue(cacheFilesStr.contains(jar3Path.toString()));
}
+ @Test
+ public void testAddActionShareLib() throws Exception {
+ Path systemLibPath = new Path(getFsTestCaseDir(), "systemlib");
+ Path javaShareLibPath = new Path(systemLibPath, "java");
+ getFileSystem().mkdirs(javaShareLibPath);
+ Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
+ getFileSystem().create(jar1Path).close();
+ Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
+ getFileSystem().create(jar2Path).close();
+
+ Path hcatShareLibPath = new Path(systemLibPath, "hcat");
+ getFileSystem().mkdirs(hcatShareLibPath);
+ Path jar3Path = new Path(hcatShareLibPath, "jar3.jar");
+ getFileSystem().create(jar3Path).close();
+ Path jar4Path = new Path(hcatShareLibPath, "jar4.jar");
+ getFileSystem().create(jar4Path).close();
+
+ Path otherShareLibPath = new Path(systemLibPath, "other");
+ getFileSystem().mkdirs(otherShareLibPath);
+ Path jar5Path = new Path(otherShareLibPath, "jar5.jar");
+ getFileSystem().create(jar5Path).close();
+
+ String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
+ "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" +
+ "<main-class>MAIN-CLASS</main-class>" +
+ "</java>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ Context context = createContext(actionXml, null);
+
+ // Test oozie server action sharelib setting
+ WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
+ XConfiguration wfConf = new XConfiguration();
+ wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
+ wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
+
+ Services.get().getConf().set("oozie.action.sharelib.for.java", "java,hcat");
+
+ JavaActionExecutor ae = new JavaActionExecutor();
+
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
+ ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
+
+ URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ String cacheFilesStr = Arrays.toString(cacheFiles);
+ assertTrue(cacheFilesStr.contains(jar1Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar2Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar3Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar4Path.toString()));
+
+ // Test per workflow action sharelib setting
+ workflow = (WorkflowJobBean) context.getWorkflow();
+ wfConf = new XConfiguration();
+ wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
+ wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ wfConf.set("oozie.action.sharelib.for.java", "other,hcat");
+ workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
+
+ Services.get().getConf().set("oozie.action.sharelib.for.java", "java");
+ ae = new JavaActionExecutor();
+
+ jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
+ ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
+
+ cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ cacheFilesStr = Arrays.toString(cacheFiles);
+ // The oozie server setting should have been overridden by workflow setting
+ assertFalse(cacheFilesStr.contains(jar1Path.toString()));
+ assertFalse(cacheFilesStr.contains(jar2Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar3Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar4Path.toString()));
+ assertTrue(cacheFilesStr.contains(jar5Path.toString()));
+ }
+
public void testAddShareLibSchemeAndAuthority() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor() {
@Override
@@ -1149,14 +1233,14 @@ public class TestJavaActionExecutor exte
JobConf conf = ae.createBaseHadoopConf(context, eActionXml);
// The next line should not throw an Exception because it will get the scheme and authority from the appPath, and not the
// sharelib path because it doesn't have a scheme or authority
- ae.addShareLib(appPath, conf, "java-action-executor");
+ ae.addShareLib(appPath, conf, new String[]{"java-action-executor"});
appPath = new Path("foo://bar:1234/blah");
conf = ae.createBaseHadoopConf(context, eActionXml);
// The next line should throw an Exception because it will get the scheme and authority from the appPath, which is obviously
// invalid, and not the sharelib path because it doesn't have a scheme or authority
try {
- ae.addShareLib(appPath, conf, "java-action-executor");
+ ae.addShareLib(appPath, conf, new String[]{"java-action-executor"});
}
catch (ActionExecutorException aee) {
assertEquals("E0902", aee.getErrorCode());
@@ -1171,7 +1255,7 @@ public class TestJavaActionExecutor exte
conf = ae.createBaseHadoopConf(context, eActionXml);
// The next line should not throw an Exception because it will get the scheme and authority from the sharelib path (and not
// from the obviously invalid appPath)
- ae.addShareLib(appPath, conf, "java-action-executor");
+ ae.addShareLib(appPath, conf, new String[]{"java-action-executor"});
}
public void testFilesystemScheme() throws Exception {
@@ -1320,21 +1404,21 @@ public class TestJavaActionExecutor exte
Element xml = XmlUtils.parseXml(str);
Path appPath = new Path(getFsTestCaseDir(), "app");
getFileSystem().mkdirs(appPath);
-
+
XConfiguration jConf = new XConfiguration();
jConf.set("p1", "v1b");
jConf.set("p3", "v3a");
OutputStream os = getFileSystem().create(new Path(appPath, "job1.xml"));
jConf.writeXml(os);
os.close();
-
+
jConf = new XConfiguration();
jConf.set("p4", "v4");
jConf.set("p3", "v3b");
os = getFileSystem().create(new Path(appPath, "job2.xml"));
jConf.writeXml(os);
os.close();
-
+
Configuration conf = new XConfiguration();
assertEquals(0, conf.size());
JavaActionExecutor.parseJobXmlAndConfiguration(createContext("<java/>", null), xml, appPath, conf);
Added: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherFSURIHandler.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherFSURIHandler.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherFSURIHandler.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherFSURIHandler.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
+import org.apache.oozie.test.XFsTestCase;
+import org.junit.Test;
+
+public class TestLauncherFSURIHandler extends XFsTestCase {
+
+ private Services services = null;
+ private JobConf conf;
+ private LauncherURIHandlerFactory uriHandlerFactory;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ services = new Services();
+ services.init();
+ URIHandlerService uriService = services.get(URIHandlerService.class);
+ uriHandlerFactory = new LauncherURIHandlerFactory(uriService.getLauncherConfig());
+ conf = createJobConf();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ services.destroy();
+ super.tearDown();
+ }
+
+ @Test
+ public void testCreate() throws Exception {
+ Path path = new Path(getFsTestCaseDir() + "/2012/12/02/");
+ LauncherURIHandler handler = uriHandlerFactory.getURIHandler(path.toUri());
+ assertTrue(handler.create(path.toUri(), conf));
+ assertTrue(getFileSystem().exists(path));
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ Path path = new Path(getFsTestCaseDir() + "/2012/12/02/");
+ LauncherURIHandler handler = uriHandlerFactory.getURIHandler(path.toUri());
+ getFileSystem().mkdirs(path);
+ assertTrue(handler.delete(path.toUri(), conf));
+ assertFalse(getFileSystem().exists(path));
+ }
+
+}
Added: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.net.URI;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.oozie.dependency.FSURIHandler;
+import org.apache.oozie.dependency.HCatURIHandler;
+import org.apache.oozie.dependency.URIHandler;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
+import org.apache.oozie.test.XHCatTestCase;
+import org.junit.Test;
+
+public class TestLauncherHCatURIHandler extends XHCatTestCase {
+
+ private Services services;
+ private URIHandlerService uriService;
+ private JobConf conf;
+ private static final String db = "db1";
+ private static final String table = "table1";
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ services = new Services();
+ services.getConf().set(URIHandlerService.URI_HANDLERS,
+ FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
+ services.init();
+ conf = createJobConf();
+ uriService = Services.get().get(URIHandlerService.class);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ services.destroy();
+ super.tearDown();
+ }
+
+ private void createTestTable() throws Exception {
+ dropTable(db, table, true);
+ dropDatabase(db, true);
+ createDatabase(db);
+ createTable(db, table, "year,month,dt,country");
+ }
+
+ private void dropTestTable() throws Exception {
+ dropTable(db, table, false);
+ dropDatabase(db, false);
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ createTestTable();
+
+ String location1 = getPartitionDir(db, table, "year=2012;month=12;dt=02;country=us");
+ String location2 = getPartitionDir(db, table, "year=2012;month=12;dt=03;country=us");
+
+ createPartitionForTestDelete(true, true);
+ URI hcatURI = getHCatURI(db, table, "year=2012;month=12;dt=02;country=us");
+ URIHandler uriHandler = uriService.getURIHandler(hcatURI);
+ assertTrue(uriHandler.exists(hcatURI, conf, getTestUser()));
+ LauncherURIHandlerFactory uriHandlerFactory = new LauncherURIHandlerFactory(uriService.getLauncherConfig());
+ LauncherURIHandler handler = uriHandlerFactory.getURIHandler(hcatURI);
+ handler.delete(hcatURI, conf);
+ assertFalse(getFileSystem().exists(new Path(location1)));
+ assertTrue(getFileSystem().exists(new Path(location2)));
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ createPartitionForTestDelete(true, false);
+ hcatURI = getHCatURI(db, table, "year=2012;month=12");
+ handler.delete(hcatURI, conf);
+ assertFalse(getFileSystem().exists(new Path(location1)));
+ assertFalse(getFileSystem().exists(new Path(location2)));
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ createPartitionForTestDelete(true, true);
+ hcatURI = getHCatURI(db, table, "month=12;country=us");
+ handler.delete(hcatURI, conf);
+ assertFalse(getFileSystem().exists(new Path(location1)));
+ assertFalse(getFileSystem().exists(new Path(location2)));
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ createPartitionForTestDelete(true, true);
+ hcatURI = getHCatURI(db, table, "country=us");
+ handler.delete(hcatURI, conf);
+ assertFalse(getFileSystem().exists(new Path(location1)));
+ assertFalse(getFileSystem().exists(new Path(location2)));
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ createPartitionForTestDelete(true, true);
+ hcatURI = getHCatURI(db, table, "dt=03");
+ handler.delete(hcatURI, conf);
+ assertTrue(getFileSystem().exists(new Path(location1)));
+ assertFalse(getFileSystem().exists(new Path(location2)));
+ assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(0, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ createPartitionForTestDelete(false, true);
+ hcatURI = getHCatURI(db, table, "dt=09");
+ handler.delete(hcatURI, conf);
+ assertTrue(getFileSystem().exists(new Path(location1)));
+ assertTrue(getFileSystem().exists(new Path(location2)));
+ assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=02;country=us").size());
+ assertEquals(1, getPartitions(db, table, "year=2012;month=12;dt=03;country=us").size());
+
+ dropTestTable();
+ }
+
+ private void createPartitionForTestDelete(boolean partition1, boolean partition2) throws Exception {
+ if (partition1) {
+ String location = addPartition(db, table, "year=2012;month=12;dt=02;country=us");
+ assertTrue(getFileSystem().exists(new Path(location)));
+ }
+ if (partition2) {
+ String location = addPartition(db, table, "year=2012;month=12;dt=03;country=us");
+ assertTrue(getFileSystem().exists(new Path(location)));
+ }
+ }
+
+}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java Mon Feb 25 21:42:07 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -29,6 +29,7 @@ import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -71,8 +72,8 @@ public class TestMapReduceActionError ex
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
- classes.add(FileSystemActions.class);
classes.add(PrepareActionsDriver.class);
+ classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java Mon Feb 25 21:42:07 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.streaming.Strea
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.HadoopAccessorService;
@@ -90,8 +91,8 @@ public class TestMapReduceActionExecutor
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
- classes.add(FileSystemActions.class);
classes.add(PrepareActionsDriver.class);
+ classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java Mon Feb 25 21:42:07 2013
@@ -18,12 +18,11 @@
package org.apache.oozie.action.hadoop;
import java.io.IOException;
-import java.util.Arrays;
-
import org.apache.hadoop.fs.Path;
import org.apache.oozie.service.Services;
import org.apache.oozie.test.XFsTestCase;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
public class TestPrepareActionsDriver extends XFsTestCase {
@@ -51,7 +50,9 @@ public class TestPrepareActionsDriver ex
fs.delete(newDir, true);
}
- PrepareActionsDriver.doOperations(Arrays.asList("hdfs"), prepareXML);
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
assertTrue(fs.exists(actionDir));
}
@@ -69,7 +70,9 @@ public class TestPrepareActionsDriver ex
try {
prepareXML = "prepare>" + "<mkdir path='" + newDir + "'/>" + "</prepare>";
- PrepareActionsDriver.doOperations(Arrays.asList("hdfs"), prepareXML);
+ JobConf conf = createJobConf();
+ LauncherMapper.setupLauncherURIHandlerConf(conf);
+ PrepareActionsDriver.doOperations(prepareXML, conf);
fail("Expected to catch an exception but did not encounter any");
} catch (LauncherException le) {
assertEquals(le.getCause().getClass(), org.xml.sax.SAXParseException.class);
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java Mon Feb 25 21:42:07 2013
@@ -35,6 +35,7 @@ import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
@@ -81,8 +82,8 @@ public class TestShellActionExecutor ext
classes.add(LauncherSecurityManager.class);
classes.add(LauncherException.class);
classes.add(LauncherMainException.class);
- classes.add(FileSystemActions.class);
classes.add(PrepareActionsDriver.class);
+ classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
classes.add(ActionStats.class);
classes.add(ActionType.class);
classes.add(LauncherMain.class);
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/client/rest/TestJsonCoordinatorAction.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/client/rest/TestJsonCoordinatorAction.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/client/rest/TestJsonCoordinatorAction.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/client/rest/TestJsonCoordinatorAction.java Mon Feb 25 21:42:07 2013
@@ -26,6 +26,7 @@ public class TestJsonCoordinatorAction e
static String CREATE_TIME = "Sat, 05 Sep 2009 00:00:00 GMT";
static String LAST_MODIFIED_TIME = "Sat, 05 Sep 2009 00:00:00 GMT";
static String missingDependencies = "a:a, a/a, a//a";
+ static String pushMissingDependencies = "hcat://a/b/c/d/f, hcat://1/2/3/4/5";
static JsonCoordinatorAction createAppAction() {
JsonCoordinatorAction app = new JsonCoordinatorAction();
@@ -40,6 +41,7 @@ public class TestJsonCoordinatorAction e
app.setStatus(CoordinatorAction.Status.WAITING);
app.setConsoleUrl("http://consoleurl:8080");
app.setMissingDependencies(missingDependencies);
+ app.setPushMissingDependencies(pushMissingDependencies);
return app;
}
@@ -56,6 +58,7 @@ public class TestJsonCoordinatorAction e
assertEquals(CoordinatorAction.Status.WAITING, app.getStatus());
assertEquals("http://consoleurl:8080", app.getConsoleUrl());
assertEquals(missingDependencies, app.getMissingDependencies());
+ assertEquals(pushMissingDependencies, app.getPushMissingDependencies());
}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java Mon Feb 25 21:42:07 2013
@@ -219,7 +219,8 @@ public class TestCoordActionInputCheckXC
fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
}
- assertEquals(";${coord:latestRange(-3,0)}", action.getMissingDependencies());
+ assertEquals(CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR + "${coord:latestRange(-3,0)}",
+ action.getMissingDependencies());
String actionXML = action.getActionXml();
String actionCreationTime = "2009-02-15T01:00" + TZ;
@@ -286,7 +287,8 @@ public class TestCoordActionInputCheckXC
fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
}
- assertEquals(";${coord:latestRange(-3,0)}", action.getMissingDependencies());
+ assertEquals(CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR + "${coord:latestRange(-3,0)}",
+ action.getMissingDependencies());
String actionXML = action.getActionXml();
String actionCreationTime = "2009-02-15T01:00" + TZ;
@@ -357,7 +359,8 @@ public class TestCoordActionInputCheckXC
fail("Action ID " + job.getId() + "@1" + " was not stored properly in db");
}
- assertEquals(";${coord:futureRange(0,3,'5')}", action.getMissingDependencies());
+ assertEquals(CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR + "${coord:futureRange(0,3,'5')}",
+ action.getMissingDependencies());
createDir(getTestCaseDir() + "/2009/03/12/");
@@ -399,16 +402,12 @@ public class TestCoordActionInputCheckXC
services = new Services();
services.init();
- // setting the configuration
- XConfiguration jobConf = new XConfiguration();
- jobConf.set(OozieClient.USER_NAME, getTestUser());
-
// setting the test path with nonExistDir
Path appPath = new Path(getFsTestCaseDir(), "coord");
String inputDir = appPath.toString() + "/coord-input/2010/07/09/01/00";
String nonExistDir = inputDir.replaceFirst("localhost", "nonExist");
try {
- caicc.pathExists(nonExistDir, jobConf);
+ caicc.pathExists(nonExistDir, new XConfiguration(), getTestUser());
fail("Should throw exception due to non-existent NN path. Therefore fail");
}
catch (IOException ioe) {
Added: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionUpdatePushMissingDependency.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionUpdatePushMissingDependency.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionUpdatePushMissingDependency.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionUpdatePushMissingDependency.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.command.coord;
+
+import java.util.Collection;
+import org.apache.oozie.CoordinatorActionBean;
+import org.apache.oozie.client.CoordinatorAction;
+import org.apache.oozie.coord.CoordELFunctions;
+import org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor;
+import org.apache.oozie.executor.jpa.JPAExecutorException;
+import org.apache.oozie.service.JPAService;
+import org.apache.oozie.service.PartitionDependencyManagerService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XDataTestCase;
+import org.apache.oozie.util.HCatURI;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCoordActionUpdatePushMissingDependency extends XDataTestCase {
+ private Services services;
+
+ @Before
+ protected void setUp() throws Exception {
+ super.setUp();
+ services = super.setupServicesForHCatalog();
+ services.init();
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ @Test
+ public void testUpdateCoordTableBasic() throws Exception {
+ String newHCatDependency = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us";
+ HCatURI hcatUri = new HCatURI(newHCatDependency);
+
+ String actionId = addInitRecords(newHCatDependency);
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
+ pdms.addMissingDependency(hcatUri, actionId);
+ pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks",
+ getPartitionMap("src=search;datastamp=12;region=us"));
+ Collection<String> availableURIs = pdms.getAvailableDependencyURIs(actionId);
+ assertEquals(availableURIs.size(), 1);
+ assertTrue(availableURIs.contains(newHCatDependency));
+
+ new CoordActionUpdatePushMissingDependency(actionId).call();
+
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+ assertNull(pdms.getAvailableDependencyURIs(actionId));
+ }
+
+ @Test
+ public void testUpdateCoordTableAdvanced() throws Exception {
+ String newHCatDependency1 = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=11;region=us";
+ String newHCatDependency2 = "hcat://hcat.server.com:5080/mydb/clicks/datastamp=12;region=us";
+ HCatURI hcatUri1 = new HCatURI(newHCatDependency1);
+ HCatURI hcatUri2 = new HCatURI(newHCatDependency2);
+
+ String fullDeps = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
+ String actionId = addInitRecords(fullDeps);
+ checkCoordAction(actionId, fullDeps, CoordinatorAction.Status.WAITING);
+
+ PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
+ pdms.addMissingDependency(hcatUri1, actionId);
+ pdms.addMissingDependency(hcatUri2, actionId);
+
+ pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks",
+ getPartitionMap("src=search;datastamp=12;region=us"));
+ Collection<String> availableURIs = pdms.getAvailableDependencyURIs(actionId);
+ assertEquals(1, availableURIs.size());
+ assertTrue(availableURIs.contains(newHCatDependency2));
+
+ new CoordActionUpdatePushMissingDependency(actionId).call();
+
+ checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
+
+ // second partition available
+
+ pdms.partitionAvailable("hcat.server.com:5080", "mydb", "clicks",
+ getPartitionMap("src=search;datastamp=11;region=us"));
+ availableURIs = pdms.getAvailableDependencyURIs(actionId);
+ assertEquals(1, availableURIs.size());
+ assertTrue(availableURIs.contains(newHCatDependency1));
+
+ new CoordActionUpdatePushMissingDependency(actionId).call();
+
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+ assertNull(pdms.getAvailableDependencyURIs(actionId));
+
+ }
+
+ private CoordinatorActionBean checkCoordAction(String actionId, String expDeps, CoordinatorAction.Status stat)
+ throws Exception {
+ try {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
+ String missDeps = action.getPushMissingDependencies();
+ assertEquals(missDeps, expDeps);
+ assertEquals(action.getStatus(), stat);
+
+ return action;
+ }
+ catch (JPAExecutorException se) {
+ throw new Exception("Action ID " + actionId + " was not stored properly in db");
+ }
+ }
+
+}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java?rev=1449911&r1=1449910&r2=1449911&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java Mon Feb 25 21:42:07 2013
@@ -22,9 +22,12 @@ import java.util.Date;
import java.util.List;
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
+import org.apache.oozie.ErrorCode;
import org.apache.oozie.SLAEventBean;
import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.CoordinatorJob.Timeunit;
+import org.apache.oozie.command.CommandException;
+import org.apache.oozie.coord.CoordELFunctions;
import org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor;
import org.apache.oozie.executor.jpa.CoordJobGetActionsJPAExecutor;
import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor;
@@ -39,21 +42,17 @@ import org.apache.oozie.test.XDataTestCa
import org.apache.oozie.util.DateUtils;
public class TestCoordMaterializeTransitionXCommand extends XDataTestCase {
- protected Services services;
@Override
protected void setUp() throws Exception {
super.setUp();
- services = new Services();
- services.init();
+ LocalOozie.start(); //LocalOozie does new Services().init();
cleanUpDBTables();
- LocalOozie.start();
}
@Override
protected void tearDown() throws Exception {
LocalOozie.stop();
- services.destroy();
super.tearDown();
}
@@ -65,6 +64,45 @@ public class TestCoordMaterializeTransit
checkCoordAction(job.getId() + "@1");
}
+ public void testActionMaterForHcatalog() throws Exception {
+ Services.get().destroy();
+ Services services = super.setupServicesForHCatalog();
+ services.init();
+ Date startTime = DateUtils.parseDateOozieTZ("2009-03-06T010:00Z");
+ Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z");
+ CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-hcat.xml",
+ CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ CoordinatorActionBean actionBean = getCoordAction(job.getId() + "@1");
+ assertEquals("file://dummyhdfs/2009/05/_SUCCESS" + CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR
+ + "${coord:latestRange(-1,0)}", actionBean.getMissingDependencies());
+
+ assertEquals("hcat://dummyhcat:1000/db1/table1/ds=2009-12" + CoordELFunctions.INSTANCE_SEPARATOR
+ + "hcat://dummyhcat:1000/db3/table3/ds=2009-05" + CoordELFunctions.INSTANCE_SEPARATOR
+ + "hcat://dummyhcat:1000/db3/table3/ds=2009-26", actionBean.getPushMissingDependencies());
+ }
+
+ public void testActionMaterForHcatalogIncorrectURI() throws Exception {
+ Services.get().destroy();
+ Services services = super.setupServicesForHCatalog();
+ services.init();
+ Date startTime = DateUtils.parseDateOozieTZ("2009-03-06T010:00Z");
+ Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z");
+ CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-neg-hcat.xml",
+ CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
+ try {
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ fail("Expected Command exception but didn't catch any");
+ }
+ catch (CommandException e) {
+ e.printStackTrace();
+ assertEquals(ErrorCode.E1001, e.getErrorCode());
+ }
+ catch (Exception e) {
+ fail("Unexpected exception " + e.getMessage());
+ }
+ }
+
public void testActionMaterWithPauseTime1() throws Exception {
Date startTime = DateUtils.parseDateOozieTZ("2009-03-06T10:00Z");
Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z");
@@ -240,6 +278,13 @@ public class TestCoordMaterializeTransit
return actionBean;
}
+ private CoordinatorActionBean getCoordAction(String actionId) throws JPAExecutorException {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ CoordinatorActionBean actionBean;
+ actionBean = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
+ return actionBean;
+ }
+
private CoordinatorJob.Status getStatus(String jobId){
CoordinatorJob job = null;
try {
@@ -284,4 +329,27 @@ public class TestCoordMaterializeTransit
fail("Action ID " + actionId + " was not stored properly in db");
}
}
+
+ private CoordinatorJobBean addRecordToCoordJobTableForWaiting(String testFileName, CoordinatorJob.Status status,
+ Date start, Date end, boolean pending, boolean doneMatd, int lastActionNum) throws Exception {
+
+ String testDir = getTestCaseDir();
+ CoordinatorJobBean coordJob = createCoordJob(testFileName, status, start, end, pending, doneMatd, lastActionNum);
+ String appXml = getCoordJobXmlForWaiting(testFileName, testDir);
+ coordJob.setJobXml(appXml);
+
+ try {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ assertNotNull(jpaService);
+ CoordJobInsertJPAExecutor coordInsertCmd = new CoordJobInsertJPAExecutor(coordJob);
+ jpaService.execute(coordInsertCmd);
+ }
+ catch (JPAExecutorException je) {
+ je.printStackTrace();
+ fail("Unable to insert the test coord job record to table");
+ throw je;
+ }
+
+ return coordJob;
+ }
}
Added: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordPushDependencyCheckXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordPushDependencyCheckXCommand.java?rev=1449911&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordPushDependencyCheckXCommand.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordPushDependencyCheckXCommand.java Mon Feb 25 21:42:07 2013
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.command.coord;
+
+import org.apache.oozie.CoordinatorActionBean;
+import org.apache.oozie.client.CoordinatorAction;
+import org.apache.oozie.coord.CoordELFunctions;
+import org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor;
+import org.apache.oozie.executor.jpa.JPAExecutorException;
+import org.apache.oozie.service.HCatAccessorService;
+import org.apache.oozie.service.JPAService;
+import org.apache.oozie.service.PartitionDependencyManagerService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XDataTestCase;
+import org.apache.oozie.util.HCatURI;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCoordPushDependencyCheckXCommand extends XDataTestCase {
+ private String server;
+ private Services services = null;
+
+ @Before
+ protected void setUp() throws Exception {
+ super.setUp();
+ services = super.setupServicesForHCatalog();
+ services.init();
+ server = getMetastoreAuthority();
+ }
+
+ @After
+ protected void tearDown() throws Exception {
+ Services.get().destroy();
+ super.tearDown();
+ }
+
+ @Test
+ public void testUpdateCoordTableSingleDep() throws Exception {
+ // Test for single dependency which is already in the hcat server
+ String db = "default";
+ String table = "tablename";
+ String newHCatDependency = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
+
+ populateTable(db, table);
+
+ String actionId = addInitRecords(newHCatDependency);
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ new CoordPushDependencyCheckXCommand(actionId).call();
+
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+
+ }
+
+ @Test
+ public void testUpdateCoordTableMultipleDepsV1() throws Exception {
+ // Test for two dependencies which are already in the hcat server
+ String db = "default";
+ String table = "tablename";
+ String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120412;country=brazil";
+ String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
+ String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
+ populateTable(db, table);
+
+ String actionId = addInitRecords(newHCatDependency);
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ new CoordPushDependencyCheckXCommand(actionId).call();
+
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+
+ }
+
+ @Test
+ public void testUpdateCoordTableMultipleDepsV2() throws Exception {
+ // Test for two dependencies : one of them is already existing in the
+ // hcat server. Other one is not.
+ // Expected to see both action in WAITING as first one is not available.
+ // Later make the other partition also available. action is expected to
+ // be READY
+ String db = "default";
+ String table = "tablename";
+ String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
+ String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
+ String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
+ populateTable(db, table);
+
+ String actionId = addInitRecords(newHCatDependency);
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ new CoordPushDependencyCheckXCommand(actionId).call();
+
+ // Checks dependencies in order. So list does not change if first one is not available
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ // Make first dependency available
+ dropPartition(db, table, "dt=20120430;country=usa");
+ addPartition(db, table, "dt=20120430;country=brazil");
+ new CoordPushDependencyCheckXCommand(actionId).call();
+ checkCoordAction(actionId, newHCatDependency2, CoordinatorAction.Status.WAITING);
+
+ addPartition(db, table, "dt=20120430;country=usa");
+ new CoordPushDependencyCheckXCommand(actionId).call();
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+ }
+
+ @Test
+ public void testUpdateCoordTableMultipleDepsV3() throws Exception {
+ // Test for two dependencies : one of them is already existing in the
+ // hcat server. Other one is not.
+ // Expected to see the action in WAITING
+ // Later make the other partition also available. action is expected to
+ // be READY
+ String db = "default";
+ String table = "tablename";
+ String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=brazil";
+ String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table + "/dt=20120430;country=usa";
+ String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
+ populateTable(db, table);
+
+ String actionId = addInitRecords(newHCatDependency);
+ checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
+
+ new CoordPushDependencyCheckXCommand(actionId, true).call();
+ checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
+ PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
+ assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
+ HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
+ assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
+
+ // Make first dependency available
+ addPartition(db, table, "dt=20120430;country=brazil");
+ new CoordPushDependencyCheckXCommand(actionId).call();
+ assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
+ checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
+ assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
+ }
+
+ private void populateTable(String db, String table) throws Exception {
+ dropTable(db, table, true);
+ dropDatabase(db, true);
+ createDatabase(db);
+ createTable(db, table, "dt,country");
+ addPartition(db, table, "dt=20120430;country=usa");
+ addPartition(db, table, "dt=20120412;country=brazil");
+ addPartition(db, table, "dt=20120413;country=brazil");
+ }
+
+ private CoordinatorActionBean checkCoordAction(String actionId, String expDeps, CoordinatorAction.Status stat)
+ throws Exception {
+ try {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
+ String missDeps = action.getPushMissingDependencies();
+ assertEquals(expDeps, missDeps);
+ assertEquals(action.getStatus(), stat);
+
+ return action;
+ }
+ catch (JPAExecutorException se) {
+ throw new Exception("Action ID " + actionId + " was not stored properly in db");
+ }
+ }
+
+}