You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2017/07/18 00:22:06 UTC
oozie git commit: OOZIE-2987 Coord action missing dependencies should
show URI template with unresolved dependencies
Repository: oozie
Updated Branches:
refs/heads/master 50b92932f -> 18694b616
OOZIE-2987 Coord action missing dependencies should show URI template with unresolved dependencies
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/18694b61
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/18694b61
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/18694b61
Branch: refs/heads/master
Commit: 18694b616f419ed958230cedb2dc0431a9f9cb5d
Parents: 50b9293
Author: puru <pu...@yahoo-inc.com>
Authored: Mon Jul 17 17:21:32 2017 -0700
Committer: puru <pu...@yahoo-inc.com>
Committed: Mon Jul 17 17:21:32 2017 -0700
----------------------------------------------------------------------
.../oozie/command/coord/CoordCommandUtils.java | 2 +-
.../dependency/CoordOldInputDependency.java | 20 ++++++++++++++++--
.../oozie/dependency/ActionDependency.java | 9 ++++++++
...tCoordActionMissingDependenciesXCommand.java | 22 ++++++++++++--------
release-log.txt | 1 +
5 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
index f7f947f..fd1230b 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
@@ -928,7 +928,7 @@ public class CoordCommandUtils {
CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
String firstMissingDependencies = coordPullInputDependency.getFirstMissingDependency();
- if (StringUtils.isEmpty(firstMissingDependencies)) {
+ if (StringUtils.isEmpty(firstMissingDependencies) || firstMissingDependencies.trim().startsWith("${coord:")) {
firstMissingDependencies = coordPushInputDependency.getFirstMissingDependency();
}
return firstMissingDependencies;
http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
index b0b6475..56aef1c 100644
--- a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
+++ b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
@@ -344,9 +344,20 @@ public class CoordOldInputDependency implements CoordInputDependency {
Map.Entry<String, ActionDependency> entry = it.next();
ActionDependency dependency = entry.getValue();
dependency.getMissingDependencies().retainAll(missingSets);
+
+ if (dependency.getUriTemplate() != null) {
+ for (int i = 0; i < dependency.getMissingDependencies().size(); i++) {
+ if (dependency.getMissingDependencies().get(i).trim().startsWith("${coord:")) {
+ dependency.getMissingDependencies().set(i,
+ dependency.getMissingDependencies().get(i) + " -> " + dependency.getUriTemplate());
+ }
+ }
+ }
+
if (dependenciesMap.get(entry.getKey()).getMissingDependencies().isEmpty()) {
it.remove();
}
+
}
return dependenciesMap;
}
@@ -376,8 +387,10 @@ public class CoordOldInputDependency implements CoordInputDependency {
}
}
if (event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace()) != null) {
+ ActionDependency unResolvedDependency = getUnResolvedDependency(coordAction, event);
dependency.getMissingDependencies()
- .addAll(getUnResolvedDependency(coordAction, event).getMissingDependencies());
+ .addAll(unResolvedDependency.getMissingDependencies());
+ dependency.setUriTemplate(unResolvedDependency.getUriTemplate());
}
dependenciesMap.put(event.getAttributeValue("name"), dependency);
}
@@ -387,10 +400,13 @@ public class CoordOldInputDependency implements CoordInputDependency {
private ActionDependency getUnResolvedDependency(CoordinatorActionBean coordAction, Element event)
throws JDOMException, URIHandlerException {
String tmpUnresolved = event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace());
- ActionDependency dependency = new ActionDependency();
StringBuilder nonResolvedList = new StringBuilder();
CoordCommandUtils.getResolvedList(getMissingDependencies(), new StringBuilder(), nonResolvedList);
+ ActionDependency dependency = new ActionDependency();
+
if (nonResolvedList.length() > 0) {
+ Element eDataset = event.getChild("dataset", event.getNamespace());
+ dependency.setUriTemplate(eDataset.getChild("uri-template", eDataset.getNamespace()).getTextTrim());
dependency.getMissingDependencies().add(tmpUnresolved);
}
return dependency;
http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java b/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java
index cc99038..a2e03c5 100644
--- a/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java
+++ b/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java
@@ -25,6 +25,7 @@ public class ActionDependency {
private List<String> missingDependencies;
private List<String> availableDependencies;
+ private String uriTemplate;
public ActionDependency() {
this(new ArrayList<String>(), new ArrayList<String>());
@@ -43,4 +44,12 @@ public class ActionDependency {
return availableDependencies;
}
+ public String getUriTemplate() {
+ return uriTemplate;
+ }
+
+ public void setUriTemplate(String uriTemplate) {
+ this.uriTemplate = uriTemplate;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
index b419a47..b5ad9cd 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java
@@ -118,7 +118,8 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase {
assertEquals(2, dependencyMap.get("F").getMissingDependencies().size());
assertEquals(dependencyMap.get("F").getMissingDependencies().get(0),
"file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS");
- assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)}");
+ assertEquals(dependencyMap.get("F").getMissingDependencies().get(1),
+ "${coord:latest(0)} -> file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}");
}
public void testCoordActionPushDependencyMissing() throws Exception {
@@ -130,14 +131,14 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase {
IOUtils.copyCharStream(reader, writer);
conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
conf.set(OozieClient.USER_NAME, getTestUser());
- String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
- String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/";
+ String datasetSuffix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa";
+ String datasetPrefix = "hcat://" + getMetastoreAuthority() + "/";
- conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix);
- conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix);
- conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix);
- conf.set("data_set_d", datasetSuffix.toString() + DB_D + datasetPrefix);
- conf.set("data_set_e", datasetSuffix.toString() + DB_E + datasetPrefix);
+ conf.set("data_set_a", datasetPrefix.toString() + DB_A + datasetSuffix);
+ conf.set("data_set_b", datasetPrefix.toString() + DB_B + datasetSuffix);
+ conf.set("data_set_c", datasetPrefix.toString() + DB_C + datasetSuffix);
+ conf.set("data_set_d", datasetPrefix.toString() + DB_D + datasetSuffix);
+ conf.set("data_set_e", datasetPrefix.toString() + DB_E + datasetSuffix);
CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
String jobId = sc.call();
@@ -145,13 +146,16 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase {
List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand(
jobId + "@1").call();
- assertEquals("${coord:latestRange(-5,0)}", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
+ assertEquals(datasetPrefix + "db_a/table1/dt=200901;country=usa",
+ CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst()));
Map<String, ActionDependency> dependencyMap = data.get(0).getSecond();
assertEquals(6, dependencyMap.size());
assertEquals(1, dependencyMap.get("A").getMissingDependencies().size());
assertEquals(6, dependencyMap.get("B").getMissingDependencies().size());
assertEquals(1, dependencyMap.get("C").getMissingDependencies().size());
+ assertEquals("${coord:latestRange(-5,0)} -> " + datasetPrefix.toString() + DB_C + datasetSuffix,
+ dependencyMap.get("C").getMissingDependencies().get(0));
assertEquals(1, dependencyMap.get("D").getMissingDependencies().size());
assertEquals(6, dependencyMap.get("E").getMissingDependencies().size());
http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f780bb2..ccb845c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-2987 Coord action missing dependencies should show URI template with unresolved dependencies
OOZIE-2004 Improve Oozie version info output (Artem Ervits via gezapeti)
OOZIE-2854 Oozie should handle transient database problems (andras.piros via gezapeti)
OOZIE-2371 Add docs for state transitions for WF Action states (daniel.becker via gezapeti)