You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by as...@apache.org on 2019/07/16 07:34:20 UTC
[oozie] branch master updated: OOZIE-3523 First missing dependency
is shown incorrectly (mgogineni via asalamon74)
This is an automated email from the ASF dual-hosted git repository.
asalamon74 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new 5e7b676 OOZIE-3523 First missing dependency is shown incorrectly (mgogineni via asalamon74)
5e7b676 is described below
commit 5e7b676a174d945076fc1923207b5aac742fde31
Author: Andras Salamon <as...@apache.org>
AuthorDate: Tue Jul 16 09:34:06 2019 +0200
OOZIE-3523 First missing dependency is shown incorrectly (mgogineni via asalamon74)
---
.../coord/CoordActionInputCheckXCommand.java | 11 ++++-
.../input/dependency/CoordOldInputDependency.java | 11 ++++-
.../coord/TestCoordActionInputCheckXCommand.java | 48 +++++++++++++++++++++-
release-log.txt | 1 +
4 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
index 3fbdae2..66875f1 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
@@ -170,8 +170,17 @@ public class CoordActionInputCheckXCommand extends CoordinatorXCommand<Void> {
LOG.info("[" + actionId + "]::CoordActionInputCheck:: Missing deps:" + firstMissingDependency + " "
+ nonResolvedList.toString());
+ boolean status = false;
+ try {
+ status = checkResolvedInput(actionXml, existList, nonExistList, actionConf);
+ }
+ catch (Exception e){
+ if(existList.length() > 0){
+ isChangeInDependency = isChangeInDependency(nonExistList, missingDependencies, nonResolvedList, status);
+ }
+ throw e;
+ }
- boolean status = checkResolvedInput(actionXml, existList, nonExistList, actionConf);
boolean isPushDependenciesMet = coordPushInputDependency.isDependencyMet();
if (status && nonResolvedList.length() > 0) {
status = (isPushDependenciesMet) ? checkUnResolvedInput(actionXml, actionConf) : false;
diff --git a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
index b25131e..296dd1a 100644
--- a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
+++ b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java
@@ -205,7 +205,16 @@ public class CoordOldInputDependency implements CoordInputDependency {
String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
for (int i = 0; i < uriList.length; i++) {
if (allExists) {
- allExists = pathExists(coordAction, uriList[i], conf, user);
+ try {
+ allExists = pathExists(coordAction, uriList[i], conf, user);
+ }
+ catch (IOException e){
+ for(int j = i; j < uriList.length; j++){
+ nonExistList.append(nonExistSeparator).append(uriList[j]);
+ nonExistSeparator = CoordELFunctions.INSTANCE_SEPARATOR;
+ }
+ throw e;
+ }
log.info("[" + coordAction.getId() + "]::ActionInputCheck:: File:" + uriList[i] + ", Exists? :"
+ allExists);
}
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
index 4ead0b6..45d3809 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java
@@ -23,7 +23,9 @@ import java.util.Arrays;
import java.util.Date;
import java.util.List;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.client.CoordinatorAction;
@@ -214,6 +216,44 @@ public class TestCoordActionInputCheckXCommand extends XDataTestCase {
}
}
+ public void testActionInputMissingDependenciesWithExceptions() throws Exception {
+ String jobId = "0000000-" + new Date().getTime() + "-TestCoordActionInputCheckXCommand-C";
+ Date startTime = DateUtils.parseDateOozieTZ("2009-02-15T23:59" + TZ);
+ Date endTime = DateUtils.parseDateOozieTZ("2009-02-16T23:59" + TZ);
+
+ //Creating the files needed for our set missing dependencies
+ Path file1 = new Path(getFsTestCaseDir(), "dir1/_SUCCESS");
+ Path file2 = new Path(getFsTestCaseDir(), "dir2/_SUCCESS");
+ Path file3 = new Path(getFsTestCaseDir(), "dir3/_SUCCESS");
+ Path file4 = new Path(getFsTestCaseDir(), "dir4/_SUCCESS");
+
+ FileSystem fs = getFileSystem();
+ fs.mkdirs(file1);
+ fs.mkdirs(file2);
+ fs.mkdirs(file3);
+ fs.mkdirs(file4);
+
+ //Setting file2 to be inaccessible
+ fs.setPermission(new Path(getFsTestCaseDir(), "dir2"), FsPermission.valueOf("----------"));
+
+ //Set missing dependencies
+ String missDeps = file1.toString() + "#" + file2.toString() + "#" + file3.toString() + "#" + file4.toString();
+ String expected = file2.toString() + "#" + file3.toString() + "#" + file4.toString();
+
+ CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-action-input-check.xml",
+ CoordinatorJob.Status.RUNNING, false, true);
+
+ CoordinatorActionBean action = addRecordToCoordActionTableForWaiting(job.getId(), 1,
+ CoordinatorAction.Status.WAITING, "coord-action-for-action-input-check.xml", missDeps);
+
+ new CoordActionInputCheckXCommand(action.getId(), job.getId()).call();
+ final JPAService jpaService = Services.get().get(JPAService.class);
+ CoordinatorActionBean caBean = jpaService.execute(new CoordActionGetJPAExecutor(action.getId()));
+ log.info("Missing deps list: " + caBean.getMissingDependencies());
+ //Checking case when second missing dependency is inaccessible to see if first one is shown or not.
+ assertEquals(expected, caBean.getMissingDependencies());
+ }
+
public void testActionInputCheckLatestActionCreationTime() throws Exception {
Services.get().getConf().setBoolean(CoordELFunctions.LATEST_EL_USE_CURRENT_TIME, false);
@@ -927,12 +967,18 @@ public class TestCoordActionInputCheckXCommand extends XDataTestCase {
}
protected CoordinatorActionBean addRecordToCoordActionTableForWaiting(String jobId, int actionNum,
CoordinatorAction.Status status, String resourceXmlName) throws Exception {
- CoordinatorActionBean action = createCoordAction(jobId, actionNum, status, resourceXmlName, 0, TZ, null);
String missDeps = getTestCaseFileUri("2009/01/29/_SUCCESS") + "#"
+ getTestCaseFileUri("2009/01/22/_SUCCESS") + "#"
+ getTestCaseFileUri("2009/01/15/_SUCCESS") + "#"
+ getTestCaseFileUri("2009/01/08/_SUCCESS");
+ return addRecordToCoordActionTableForWaiting(jobId, actionNum, status, resourceXmlName, missDeps);
+ }
+
+ protected CoordinatorActionBean addRecordToCoordActionTableForWaiting(String jobId, int actionNum,
+ CoordinatorAction.Status status, String resourceXmlName, String missDeps) throws Exception {
+ CoordinatorActionBean action = createCoordAction(jobId, actionNum, status, resourceXmlName, 0, TZ, null);
+
action.setMissingDependencies(missDeps);
try {
diff --git a/release-log.txt b/release-log.txt
index 62634fe..1a68737 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-3523 First missing dependency is shown incorrectly (mgogineni via asalamon74)
OOZIE-2836 Remove .ps1 and .cmd windows scripts (kmarton via asalamon74)
OOZIE-3506 Flaky test TestOozieRollingPolicy (asalamon74 via kmarton)
OOZIE-3476 Migrate classes without setup/tearDown to JUnit4 (asalamon74 via kmarton)