You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:25 UTC
[20/50] [abbrv] oozie git commit: OOZIE-2770 Show missing
dependencies for coord actions
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
new file mode 100644
index 0000000..b419a47
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
@@ -0,0 +1,288 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.command.coord;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.CoordinatorActionBean;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.coord.input.logic.TestCoordInputLogicPush;
+import org.apache.oozie.coord.input.logic.TestCoordInputLogicPush.TEST_TYPE;
+import org.apache.oozie.dependency.ActionDependency;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.test.XHCatTestCase;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.Pair;
+import org.apache.oozie.util.XConfiguration;
+
+public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase {
+ private Services services;
+ final String TABLE = "table1";
+ final String DB_A = "db_a";
+ final String DB_B = "db_b";
+ final String DB_C = "db_c";
+ final String DB_D = "db_d";
+ final String DB_E = "db_e";
+ final String DB_F = "db_f";
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ services = super.setupServicesForHCatalog();
+ services.init();
+
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ services.destroy();
+ super.tearDown();
+ }
+
+ public void testCoordActionPullDependencyMissing() throws Exception {
+
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+
+ // CASE 1: Failure case i.e. multiple data-in instances
+ Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
+ Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
+ IOUtils.copyCharStream(reader, writer);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("data_set_a", "file://" + getTestCaseDir() + "/input-data/a/${YEAR}/${DAY}");
+ conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b/${YEAR}/${DAY}");
+ conf.set("data_set_c", "file://" + getTestCaseDir() + "/input-data/c/${YEAR}/${DAY}");
+ conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
+ conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
+
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+ new CoordMaterializeTransitionXCommand(jobId, 3600).call();
+ List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(
+ jobId + "@1").call();
+
+ Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
+ assertEquals(6, dependencyMap.size());
+ assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
+ createTestCaseSubDir("input-data/a/2009/01/_SUCCESS".split("/"));
+ createTestCaseSubDir("input-data/b/2009/01/_SUCCESS".split("/"));
+ new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
+ data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
+ dependencyMap = data.get(0).getSecond();
+ assertEquals(5, dependencyMap.size());
+ assertNull(dependencyMap.get("A"));
+ assertEquals(5, dependencyMap.get("B").getMissingDependencies().size());
+ assertTrue(dependencyMap.get("B").getMissingDependencies()
+ .contains("file://" + getTestCaseDir() + "/input-data/b/2009/31/_SUCCESS"));
+ assertTrue(dependencyMap.get("B").getMissingDependencies()
+ .contains("file://" + getTestCaseDir() + "/input-data/b/2009/30/_SUCCESS"));
+ assertTrue(dependencyMap.get("B").getMissingDependencies()
+ .contains("file://" + getTestCaseDir() + "/input-data/b/2009/29/_SUCCESS"));
+ assertTrue(dependencyMap.get("B").getMissingDependencies()
+ .contains("file://" + getTestCaseDir() + "/input-data/b/2009/28/_SUCCESS"));
+ assertTrue(dependencyMap.get("B").getMissingDependencies()
+ .contains("file://" + getTestCaseDir() + "/input-data/b/2009/27/_SUCCESS"));
+ assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
+ assertEquals(dependencyMap.get("D").getMissingDependencies().get(0),
+ "file://" + getTestCaseDir() + "/input-data/d/2009/01");
+
+ assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
+
+ assertEquals(2, dependencyMap.get("F").getMissingDependencies().size());
+ assertEquals(dependencyMap.get("F").getMissingDependencies().get(0),
+ "file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS");
+ assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)}");
+ }
+
+ public void testCoordActionPushDependencyMissing() throws Exception {
+ createTestTables();
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
+ Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
+ IOUtils.copyCharStream(reader, writer);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
+ String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/";
+
+ conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix);
+ conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix);
+ conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix);
+ conf.set("data_set_d", datasetSuffix.toString() + DB_D + datasetPrefix);
+ conf.set("data_set_e", datasetSuffix.toString() + DB_E + datasetPrefix);
+
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+ new CoordMaterializeTransitionXCommand(jobId, 3600).call();
+ List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(
+ jobId + "@1").call();
+
+ assertEquals("${coord:latestRange(-5,0)}", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
+
+ Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
+ assertEquals(6, dependencyMap.size());
+ assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
+
+ addPartition(DB_A, TABLE, "dt=200901;country=usa");
+ new CoordPushDependencyCheckXCommand(jobId + "@1").call();
+ data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
+ dependencyMap = data.get(0).getSecond();
+ assertEquals(5, dependencyMap.size());
+ assertNull(dependencyMap.get("A"));
+
+ addPartition(DB_B, TABLE, "dt=200901;country=usa");
+ addPartition(DB_B, TABLE, "dt=200931;country=usa");
+ addPartition(DB_B, TABLE, "dt=200930;country=usa");
+ addPartition(DB_B, TABLE, "dt=200929;country=usa");
+ addPartition(DB_B, TABLE, "dt=200928;country=usa");
+ addPartition(DB_B, TABLE, "dt=200927;country=usa");
+
+ new CoordPushDependencyCheckXCommand(jobId + "@1").call();
+ data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
+ dependencyMap = data.get(0).getSecond();
+ assertEquals(4, dependencyMap.size());
+ assertNull(dependencyMap.get("B"));
+ }
+
+ public void testCoordActionPullPushDependencyMissing() throws Exception {
+ createTestTables();
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1);
+ Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
+ IOUtils.copyCharStream(reader, writer);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
+ String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/";
+
+ conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix);
+ conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix);
+ conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix);
+ conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}");
+ conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
+
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+ new CoordMaterializeTransitionXCommand(jobId, 3600).call();
+ List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(
+ jobId + "@1").call();
+
+ assertEquals("file://" + getTestCaseDir() + "/input-data/d/2009/01",
+ CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
+
+ Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
+ assertEquals(6, dependencyMap.size());
+ assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
+ assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
+ assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
+
+ addPartition(DB_A, TABLE, "dt=200901;country=usa");
+ addPartition(DB_B, TABLE, "dt=200901;country=usa");
+ addPartition(DB_B, TABLE, "dt=200931;country=usa");
+ addPartition(DB_B, TABLE, "dt=200930;country=usa");
+ addPartition(DB_B, TABLE, "dt=200929;country=usa");
+ addPartition(DB_B, TABLE, "dt=200928;country=usa");
+ addPartition(DB_B, TABLE, "dt=200927;country=usa");
+ createTestCaseSubDir("input-data/d/2009/01".split("/"));
+
+ new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
+ new CoordPushDependencyCheckXCommand(jobId + "@1").call();
+ data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
+ dependencyMap = data.get(0).getSecond();
+ assertEquals(3, dependencyMap.size());
+ assertNull(dependencyMap.get("B"));
+ assertNull(dependencyMap.get("D"));
+ }
+
+ public void testCoordActionInputLogicMissing() throws Exception {
+ createTestTables();
+
+ Configuration conf = TestCoordInputLogicPush.getConfForCombine("file://" + getTestCaseDir(),
+ "hcat://" + getMetastoreAuthority());
+ conf.set("initial_instance_a", "2014-10-07T00:00Z");
+ conf.set("initial_instance_b", "2014-10-07T00:00Z");
+
+ String inputLogic =
+ // @formatter:off
+ "<and name=\"test\">" + "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</and>";
+ // @formatter:on
+ String jobId = TestCoordInputLogicPush.submitCoord(getTestCaseDir(), "coord-inputlogic-combine.xml", conf,
+ inputLogic, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_RANGE,
+ TEST_TYPE.LATEST_RANGE);
+
+ new CoordMaterializeTransitionXCommand(jobId, 3600).call();
+ List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(
+ jobId + "@1").call();
+ Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
+ assertEquals(6, dependencyMap.size());
+
+ assertNull(CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
+
+ createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/"));
+ addPartition(DB_A, TABLE, "dt=20141008;country=usa");
+
+ new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
+ new CoordPushDependencyCheckXCommand(jobId + "@1").call();
+ new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
+
+ data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call();
+ dependencyMap = data.get(0).getSecond();
+ assertEquals(4, dependencyMap.size());
+ assertNull(dependencyMap.get("A"));
+ assertNull(dependencyMap.get("B"));
+
+ }
+
+ private void createSingleTestDB(String db) throws Exception {
+ dropTable(db, TABLE, true);
+ dropDatabase(db, true);
+ createDatabase(db);
+ createTable(db, TABLE, "dt,country");
+ }
+
+ private void createTestTables() throws Exception {
+ createSingleTestDB(DB_A);
+ createSingleTestDB(DB_B);
+ createSingleTestDB(DB_C);
+ createSingleTestDB(DB_D);
+ createSingleTestDB(DB_E);
+ createSingleTestDB(DB_F);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java b/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java
index 6684a1f..d66aace 100644
--- a/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java
+++ b/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java
@@ -64,9 +64,10 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
private String server;
private static final String table = "table1";
- final long TIME_DAYS = 60 * 60 * 1000 * 24;
- enum TEST_TYPE {
+ public final static long TIME_DAYS = 60 * 60 * 1000 * 24;
+
+ public static enum TEST_TYPE {
CURRENT_SINGLE, CURRENT_RANGE, LATEST_SINGLE, LATEST_RANGE;
};
@@ -132,7 +133,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"</or>";
//@formatter:on
conf.set("partitionName", "test");
- String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
+ String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
String input = addPartition("db_b", "table1", "dt=20141008;country=usa");
@@ -168,7 +169,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"</and>";
//@formatter:on
conf.set("partitionName", "test");
- final String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
+ final String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
String input1 = addPartition("db_a", "table1", "dt=20141008;country=usa");
String input2 = addPartition("db_b", "table1", "dt=20141008;country=usa");
@@ -215,7 +216,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"</and>";
//@formatter:on
conf.set("partitionName", "test");
- final String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_RANGE,
+ final String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_RANGE,
TEST_TYPE.LATEST_RANGE);
List<String> inputPartition = createPartitionWithTime("db_a", now, 0, 1, 2);
inputPartition.addAll(createPartitionWithTime("db_c", now, 0, 1, 2));
@@ -246,7 +247,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"<data-in dataset=\"B\" />" +
"</and>";
//@formatter:on
- String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
+ String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5);
inputDir.addAll(createPartitionWithTime("db_a", now, 0, 1, 2, 3, 4, 5));
@@ -280,7 +281,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"<data-in dataset=\"B\"/>" +
"</and>";
//@formatter:on
- String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE,
+ String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE,
TEST_TYPE.CURRENT_RANGE);
List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5);
@@ -322,7 +323,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"</or>";
//@formatter:on
- String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
+ String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5);
inputDir.addAll(createPartitionWithTime("db_a", now, 0, 1, 2, 3, 4, 5));
@@ -351,7 +352,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"<data-in name=\"testB\" dataset=\"B\" />" +
"</and>";
//@formatter:on
- String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
+ String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
String input1 = createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/"));
String input2 = addPartition("db_a", "table1", "dt=20141008;country=usa");
@@ -394,7 +395,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
"</and>";
//@formatter:on
- String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
+ String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE);
String input1 = createTestCaseSubDir(("input-data/d/" + sd.format(now) + "/_SUCCESS").split("/"));
sd = new SimpleDateFormat("yyyyMMdd");
@@ -429,31 +430,31 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
conf.set("db_e", "db_e");
conf.set("db_f", "db_f");
conf.set("table", table);
- conf.set("wfPath", getWFPath());
+ conf.set("wfPath", getWFPath(getTestCaseFileUri("workflow.xml")));
conf.set("partitionName", "test");
return conf;
+ }
+ public Configuration getConfForCombine() throws Exception {
+ return getConfForCombine("file://" + getTestCaseDir(), "hcat://" + getMetastoreAuthority());
}
- private Configuration getConfForCombine() throws Exception {
+ public static Configuration getConfForCombine(String testCaseDir, String hcatURL) throws Exception {
Configuration conf = new XConfiguration();
conf.set("start_time", "2014-10-08T00:00Z");
conf.set("end_time", "2015-10-08T00:00Z");
conf.set("initial_instance", "2014-10-08T00:00Z");
- conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b");
- conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d");
- conf.set("data_set_f", "file://" + getTestCaseDir() + "/input-data/f");
+ conf.set("data_set_b", testCaseDir + "/input-data/b");
+ conf.set("data_set_d", testCaseDir + "/input-data/d");
+ conf.set("data_set_f", testCaseDir + "/input-data/f");
conf.set("start_time", "2014-10-08T00:00Z");
conf.set("end_time", "2015-10-08T00:00Z");
conf.set("initial_instance_a", "2014-10-08T00:00Z");
conf.set("initial_instance_b", "2014-10-08T00:00Z");
-
- String dataset1 = "hcat://" + getMetastoreAuthority();
-
- conf.set("data_set", dataset1.toString());
+ conf.set("data_set", hcatURL);
conf.set("db_a", "db_a");
conf.set("db_b", "db_b");
conf.set("db_c", "db_c");
@@ -461,16 +462,19 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
conf.set("db_e", "db_e");
conf.set("db_f", "db_f");
conf.set("table", table);
- conf.set("wfPath", getWFPath());
+ conf.set("wfPath", getWFPath(testCaseDir + "/workflow.xml"));
conf.set("partitionName", "test");
-
return conf;
-
}
- private String _testCoordSubmit(String coordinatorXml, Configuration conf, String inputLogic, TEST_TYPE... testType)
+ public String submitCoord(String coordinatorXml, Configuration conf, String inputLogic, TEST_TYPE... testType)
throws Exception {
- String appPath = "file://" + getTestCaseDir() + File.separator + "coordinator.xml";
+ return submitCoord(getTestCaseDir(), coordinatorXml, conf, inputLogic, testType);
+ }
+
+ public static String submitCoord(String testCaseDir, String coordinatorXml, Configuration conf, String inputLogic,
+ TEST_TYPE... testType) throws Exception {
+ String appPath = "file://" + testCaseDir + File.separator + "coordinator.xml";
String content = IOUtils.getResourceAsString(coordinatorXml, -1);
content = content.replaceAll("=input-logic=", inputLogic);
@@ -504,8 +508,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
return coordId;
}
- public String getWFPath() throws Exception {
- String workflowUri = getTestCaseFileUri("workflow.xml");
+ public static String getWFPath(String workflowUri) throws Exception {
String appXml = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='map-reduce-wf'> " + "<start to='end' /> "
+ "<end name='end' /> " + "</workflow-app>";
@@ -513,7 +516,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
return workflowUri;
}
- private void writeToFile(String appXml, String appPath) throws IOException {
+ private static void writeToFile(String appXml, String appPath) throws IOException {
File wf = new File(URI.create(appPath));
PrintWriter out = null;
try {
@@ -593,7 +596,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase {
return conf;
}
- private String getEnumText(TEST_TYPE testType) {
+ private static String getEnumText(TEST_TYPE testType) {
switch (testType) {
case LATEST_SINGLE:
return "<instance>\\${coord:latest(0)}</instance>";
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java
index 0e74f2d..4fc8653 100644
--- a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java
+++ b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java
@@ -39,8 +39,11 @@ import org.apache.oozie.client.CoordinatorAction;
import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.CoordinatorJob.Execution;
import org.apache.oozie.client.rest.RestConstants;
+import org.apache.oozie.command.CommandException;
+import org.apache.oozie.dependency.ActionDependency;
import org.apache.oozie.service.CoordinatorEngineService;
import org.apache.oozie.util.DateUtils;
+import org.apache.oozie.util.Pair;
public class MockCoordinatorEngineService extends CoordinatorEngineService {
public static final String JOB_ID = "coord-job-C-";
@@ -250,6 +253,13 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService {
}
@Override
+ public List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> getCoordActionMissingDependencies(
+ String id, String actions, String dates) throws CommandException {
+ did = RestConstants.COORD_ACTION_MISSING_DEPENDENCIES;
+ return new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>() ;
+ }
+
+ @Override
public CoordinatorJobInfo suspendJobs(String filter, int start, int length)
throws CoordinatorEngineException {
did = RestConstants.JOBS;
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/resources/coord-multiple-output-instance5.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-output-instance5.xml b/core/src/test/resources/coord-multiple-output-instance5.xml
new file mode 100644
index 0000000..5932860
--- /dev/null
+++ b/core/src/test/resources/coord-multiple-output-instance5.xml
@@ -0,0 +1,108 @@
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ -->
+<coordinator-app xmlns="uri:oozie:coordinator:0.5" name="NAME"
+ frequency="${coord:days(1)}" start="2009-02-01T01:00Z" end="2009-02-03T23:59Z"
+ timezone="UTC">
+ <controls>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <datasets>
+
+ <dataset name="a" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T01:00Z" timezone="UTC">
+ <uri-template>${data_set_a}
+ </uri-template>
+ </dataset>
+
+ <dataset name="b" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T01:00Z" timezone="UTC">
+ <uri-template>${data_set_b}
+ </uri-template>
+ </dataset>
+ <dataset name="c" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T01:00Z" timezone="UTC">
+ <uri-template>${data_set_c}
+ </uri-template>
+ </dataset>
+ <dataset name="d" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T01:00Z" timezone="UTC">
+ <uri-template>${data_set_d}
+ </uri-template>
+ <done-flag></done-flag>
+ </dataset>
+
+ <dataset name="e" frequency="${coord:days(1)}"
+ initial-instance="2009-01-01T01:00Z" timezone="UTC">
+ <uri-template>${data_set_e}
+ </uri-template>
+ </dataset>
+
+ <dataset name="local_a" frequency="${coord:days(1)}"
+ initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}
+ </uri-template>
+ </dataset>
+ </datasets>
+ <input-events>
+ <data-in name="A" dataset="a">
+ <instance>${coord:current(0)}</instance>
+ </data-in>
+ <data-in name="B" dataset="b">
+ <start-instance>${coord:current(-5)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ <data-in name="C" dataset="c">
+ <start-instance>${coord:latest(-5)}</start-instance>
+ <end-instance>${coord:latest(0)}</end-instance>
+ </data-in>
+ <data-in name="D" dataset="d">
+ <instance>${coord:current(0)}</instance>
+ </data-in>
+ <data-in name="E" dataset="e">
+ <start-instance>${coord:current(-5)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ <data-in name="F" dataset="e">
+ <instance>${coord:current(0)}</instance>
+ <instance>${coord:latest(0)}</instance>
+ </data-in>
+ </input-events>
+ <output-events>
+ <data-out name="LOCAL_A" dataset="local_a">
+ <instance>${coord:current(0)}</instance>
+ </data-out>
+ </output-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>inputA</name>
+ <value>${coord:dataIn('A')}</value>
+ </property>
+ <property>
+ <name>inputB</name>
+ <value>${coord:dataOut('LOCAL_A')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/docs/src/site/twiki/DG_CommandLineTool.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki
index 2dbbd4c..67838fc 100644
--- a/docs/src/site/twiki/DG_CommandLineTool.twiki
+++ b/docs/src/site/twiki/DG_CommandLineTool.twiki
@@ -999,6 +999,31 @@ All other arguments are optional:
$oozie job -slaenable <bundle_job_id> [-action 1,3-4,7-40] [-date 2009-01-01T01:00Z::2009-05-31T23:59Z,2009-11-10T01:00Z::2009-12-31T22:00Z] [-coordinator <List_of_coord_names/ids]
</verbatim>
+---+++ Getting missing dependencies of coordinator action(s)
+ * Coordination action id can be specified directly for getting missing dependencies of a single action.
+ * To get information on multiple actions, either -action or -date option can be specified with the coordinator job id.
+ * missingdeps command doesn't recompute dependencies. It list missing dependencies which were last computed.
+ * Oozie checks missing dependencies sequentially, and it will stop on first missing dependency. =Blocked On= is the first missing dependency for action. So, there could be a chance that Oozie will report some missing dependencies, but it might be present. To resolve the waiting issue, one should fix the blockedOn missing dependency.
+ * For input logic, missingdeps command doesn't compute input-logic expression. It will report everything which is missing or not computed.
+<verbatim>
+oozie job -oozie http://localhost:11000/oozie -missingDependencies 0000000-170104141851590-oozie-puru-C -action 1
+oozie job -oozie http://localhost:11000/oozie -missingDependencies 0000000-170104141851590-oozie-puru-C@1
+
+id : 1
+blockedOn : hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS
+
+dataSet : input-1
+missingDependencies :
+ hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS
+
+dataSet : input-2
+missingDependencies :
+ hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS
+ hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/40/_SUCCESS
+ hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/20/_SUCCESS
+ hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS
+$
+</verbatim>
---++ Jobs Operations
---+++ Checking the Status of multiple Workflow Jobs
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/docs/src/site/twiki/WebServicesAPI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki
index 8406da6..03f05d2 100644
--- a/docs/src/site/twiki/WebServicesAPI.twiki
+++ b/docs/src/site/twiki/WebServicesAPI.twiki
@@ -1616,6 +1616,55 @@ PUT /oozie/v2/job/0000003-140319184715726-oozie-puru-C?action=sla-enable&action-
</verbatim>
Will enable SLA alert for actions 1,14,17,18,19,20.
+---+++ Getting missing dependencies of coordinator action(s)
+
+<verbatim>
+GET oozie/v2/job/0000000-170104115137443-oozie-puru-C?show=missing-dependencies&action-list=1,20
+</verbatim>
+
+*Response*
+
+<verbatim>
+HTTP/1.1 200 OK
+Content-Type: application/json;charset=UTF-8
+
+{
+"missingDependencies":
+[{
+ "blockedOn": "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS",
+ "dataSets":
+ [
+ {
+ "missingDependencies":
+ [
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS"
+ ],
+ "dataSet": "input-2"
+ }
+ ],
+ "id": 1
+ },
+ {
+ "blockedOn": "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/20/00/_SUCCESS",
+ "dataSets":
+ [
+ {
+ "missingDependencies":
+ [
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/20/00/_SUCCESS",
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/40/_SUCCESS",
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/20/_SUCCESS",
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/00/_SUCCESS",
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/18/40/_SUCCESS",
+ "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/18/20/_SUCCESS"
+ ],
+ "dataSet": "input-2"
+ }
+ ],
+ "id": 20
+ }]
+}
+</verbatim>
---++++ Jobs Information
A HTTP GET request retrieves workflow and coordinator jobs information.
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index cd40ea6..081ead8 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2770 Show missing dependencies for coord actions (puru)
OOZIE-2630 Oozie Coordinator EL Functions to get first day of the week/month (satishsaley)
OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna)
OOZIE-2619 Make Hive action defaults to match hive defaults when running from command line (venkatnrangan via abhishekbafna)
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/webapp/src/main/webapp/oozie-console.js
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js
index 7b20e91..38989c7 100644
--- a/webapp/src/main/webapp/oozie-console.js
+++ b/webapp/src/main/webapp/oozie-console.js
@@ -1484,10 +1484,9 @@ function coordJobDetailsPopup(response, request) {
var actionStatus = thisGrid.store.data.items[rowIndex].data;
var workflowId = actionStatus["externalId"];
if(workflowId == null) {
- jobDetailsTab.getComponent('coord_job_log').show();
- Ext.getCmp('actions_text_box').setValue(actionStatus["id"].split("@")[1]);
- Ext.getCmp('search_filter_box').setValue('recent=5m');
- fetchLogs(coordJobId, actionsTextBox.getValue());
+ jobDetailsTab.getComponent('coord_action_missing_dependencies').show();
+ Ext.getCmp('actions_missing_dependencies').setValue(actionStatus["id"].split("@")[1]);
+ viewMissingDependencies.execute();
}
else {
jobDetailsGridWindow(workflowId);
@@ -1681,6 +1680,114 @@ function coordJobDetailsPopup(response, request) {
},
frame: false
});
+
+ var missingDependenciesTreeRoot = new Ext.tree.TreeNode({
+ text: "Coord Action Missing Dependencies",
+ expanded: true
+ });
+
+ var missingDependenciesTabButton = new Ext.Button({
+ text: 'Get Missing Dependencies',
+ ctCls: 'x-btn-over',
+ ctCls: 'spaces',
+ isFormField: true,
+ handler: function() {
+ viewMissingDependencies.execute();
+ }
+ });
+
+ var missingDependenciesActionText = new Ext.form.Label({
+ text : 'Enter Coordinator Action number : ',
+ ctCls: 'spaces'
+ });
+ var missingDependenciesTextBox = new Ext.form.TextField({
+ fieldLabel: 'missingDependenciesAction',
+ id: 'actions_missing_dependencies',
+ name: 'missingDependenciesAction',
+ width: 150,
+ value: ''
+ });
+
+ var viewMissingDependencies = new Ext.Action({
+ text: " ",
+ icon: 'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+ Ext.Ajax.request({
+ url: getOozieBase() + 'job/' + coordJobId + '?show=missing-dependencies&action-list=' +
+ missingDependenciesTextBox.getValue(),
+ success: function(response, request) {
+ var jsonData = JSON.parse(response.responseText);
+ var missingDependencies = jsonData["missingDependencies"];
+ while (missingDependenciesTreeRoot.hasChildNodes()) {
+ var child = missingDependenciesTreeRoot.firstChild;
+ missingDependenciesTreeRoot.removeChild(child);
+ }
+ var missingDependenciesTree = treeNodeForMissingDependencies(missingDependencies, null, null,
+ missingDependenciesTreeRoot);
+ missingDependenciesTreeRoot.expand(false, true);
+ },
+ failure : function(response, request) {
+ Ext.Msg.minWidth = 360;
+ Ext.Msg.alert(response.getResponseHeader["oozie-error-message"]);
+ }
+ });
+ }
+ });
+
+ function treeNodeForMissingDependencies(json, rootText, blockedOn, rootNode) {
+ var result;
+ if( rootNode ){
+ result = rootNode;
+ } else {
+ result = new Ext.tree.TreeNode({
+ text: rootText
+ });
+ }
+ if (typeof json === 'object') {
+ for (var i in json) {
+ if (json[i]) {
+ if (typeof json[i] == 'object') {
+ var c;
+ if (json[i]['id']) {
+ c = treeNodeForMissingDependencies(json[i]['dataSets'], json[i]['id'], json[i]['blockedOn']);
+ }
+ if (json[i]['dataSet']) {
+ c = treeNodeForMissingDependencies(json[i]['missingDependencies'], json[i]['dataSet'], blockedOn);
+ if( blockedOn ) {
+ var blockedOnChild = new Ext.tree.TreeNode({
+ text: "Blocked On"
+ });
+ blockedOnChild.appendChild(new Ext.tree.TreeNode({
+ text: blockedOn
+ }));
+ result.appendChild(blockedOnChild);
+ blockedOn = null;
+ }
+ }
+ if (c) {
+ result.appendChild(c);
+ }
+ }
+ else if (typeof json[i] != 'function') {
+ result.appendChild(new Ext.tree.TreeNode({
+ text: json[i]
+ }));
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ var missingDependenciesArea = new Ext.tree.TreePanel({
+ autoScroll: true,
+ useArrows: true,
+ height: 430,
+ deferredRender: false,
+ root: missingDependenciesTreeRoot,
+ tbar: [missingDependenciesActionText, missingDependenciesTextBox, missingDependenciesTabButton],
+ });
+
function populateReruns(coordActionId) {
var actionNum = rerunActionTextBox.getValue();
store.baseParams.scope = actionNum;
@@ -1698,6 +1805,7 @@ function coordJobDetailsPopup(response, request) {
var jobDetailsTab = new Ext.TabPanel({
activeTab: 0,
autoHeight: true,
+ layoutOnTabChange: true,
deferredRender: false,
items: [ {
title: 'Coord Job Info',
@@ -1751,6 +1859,11 @@ function coordJobDetailsPopup(response, request) {
tbar: [
rerunActionText, rerunActionTextBox, getRerunsButton]
},
+ {
+ title: 'Action Missing Dependencies',
+ id: 'coord_action_missing_dependencies',
+ items: missingDependenciesArea
+ }
]});
jobDetailsTab.addListener("tabchange", function(panel, selectedTab) {
@@ -3169,8 +3282,8 @@ function initConsole() {
viewConfig.execute();
serverVersion.execute();
if (isInstrumentationServiceEnabled == "true") {
- viewInstrumentation.execute();
- }
+ viewInstrumentation.execute();
+ }
if (isMetricsInstrumentationServiceEnabled == "true") {
viewMetrics.execute();
}