You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by sr...@apache.org on 2016/01/20 18:05:34 UTC
[08/50] [abbrv] tez git commit: TEZ-3017. HistoryACLManager does not
have a close method for cleanup (bikas)
TEZ-3017. HistoryACLManager does not have a close method for cleanup (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9816a49b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9816a49b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9816a49b
Branch: refs/heads/TEZ-2980
Commit: 9816a49bba0c89651f9b5bb66ac20f4eb0364ef2
Parents: 5334d62
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Dec 29 01:54:00 2015 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Dec 29 01:54:00 2015 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../src/main/java/org/apache/tez/client/TezClient.java | 4 ++++
.../tez/common/security/HistoryACLPolicyManager.java | 5 ++++-
.../test/java/org/apache/tez/client/TestTezClient.java | 6 +++++-
.../dag/history/ats/acls/ATSHistoryACLPolicyManager.java | 11 ++++++++---
.../tez/dag/history/ats/acls/TestATSHistoryWithACLs.java | 9 ++++++---
.../history/logging/ats/ATSHistoryLoggingService.java | 3 +++
7 files changed, 32 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 546c3fd..ce35fd1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -11,6 +11,7 @@ INCOMPATIBLE CHANGES
TEZ-2972. Avoid task rescheduling when a node turns unhealthy
ALL CHANGES:
+ TEZ-3017. HistoryACLManager does not have a close method for cleanup
TEZ-2914. Ability to limit vertex concurrency
TEZ-3011. Link Vertex Name in Dag Tasks/Task Attempts to Vertex
TEZ-3006. Remove unused import in TestHistoryParser.
@@ -300,6 +301,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES
+ TEZ-3017. HistoryACLManager does not have a close method for cleanup
TEZ-2914. Ability to limit vertex concurrency
TEZ-2918. Make progress notifications in IOs
TEZ-2952. NPE in TestOnFileUnorderedKVOutput
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index ff131e0..fc98d1a 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -558,6 +558,10 @@ public class TezClient {
*/
public synchronized void stop() throws TezException, IOException {
try {
+ if (historyACLPolicyManager != null) {
+ historyACLPolicyManager.close();
+ }
+
if (sessionStarted) {
LOG.info("Shutting down Tez Session"
+ ", sessionName=" + clientName
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
index dea89cc..fc0f57c 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
@@ -69,5 +69,8 @@ public interface HistoryACLPolicyManager extends Configurable {
public void updateTimelineEntityDomain(Object timelineEntity, String domainId);
-
+ /**
+ * Call this to stop and clean up
+ */
+ public void close();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 9adb9bd..c2531c6 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.counters.LimitExceededException;
import org.apache.tez.common.counters.Limits;
import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.common.security.HistoryACLPolicyManager;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.PreWarmVertex;
import org.apache.tez.dag.api.ProcessorDescriptor;
@@ -162,10 +163,12 @@ public class TestTezClient {
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
-
+ HistoryACLPolicyManager mockAcl = mock(HistoryACLPolicyManager.class);
+
ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.RUNNING);
+ client.setUpHistoryAclManager(mockAcl);
client.start();
verify(client.mockYarnClient, times(1)).init((Configuration)any());
verify(client.mockYarnClient, times(1)).start();
@@ -263,6 +266,7 @@ public class TestTezClient {
(ShutdownSessionRequestProto) any());
}
verify(client.mockYarnClient, times(1)).stop();
+ verify(mockAcl, times(1)).close();
}
@Test (timeout=5000)
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
index 3fa3db6..91ffe7b 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.service.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.security.ACLConfigurationParser;
import org.apache.tez.common.security.ACLManager;
import org.apache.tez.common.security.ACLType;
@@ -43,8 +43,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
-import com.google.common.annotations.VisibleForTesting;
-
public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager {
private final static Logger LOG = LoggerFactory.getLogger(ATSHistoryACLPolicyManager.class);
@@ -250,4 +248,11 @@ public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager {
entity.setDomainId(domainId);
}
+ @Override
+ public void close() {
+ if (timelineClient != null && timelineClient.isInState(Service.STATE.STARTED)) {
+ timelineClient.stop();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
index eaf24d3..512913d 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.*;
import java.io.IOException;
import java.util.Collection;
@@ -74,9 +75,6 @@ import com.sun.jersey.api.client.WebResource;
import org.mockito.Matchers;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.doThrow;
-
public class TestATSHistoryWithACLs {
private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithACLs.class);
@@ -385,7 +383,9 @@ public class TestATSHistoryWithACLs {
}
dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED);
Assert.assertNull(dagLogging);
+ myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient);
tezSession.stop();
+ verify(myAclPolicyManager.timelineClient, times(1)).stop();
}
/**
@@ -465,7 +465,10 @@ public class TestATSHistoryWithACLs {
}
dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED);
Assert.assertNull(dagLogging);
+ myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient);
tezClient.stop();
+ verify(myAclPolicyManager.timelineClient, times(1)).stop();
+
}
/**
* Test Disable Logging for all dags in a session
http://git-wip-us.apache.org/repos/asf/tez/blob/9816a49b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index 6ea21e2..a66da24 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -243,6 +243,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
+ ", eventQueueBacklog=" + eventQueue.size());
}
timelineClient.stop();
+ if (historyACLPolicyManager != null) {
+ historyACLPolicyManager.close();
+ }
}
private void getEventBatch(List<DAGHistoryEvent> events) throws InterruptedException {