You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2017/02/03 21:46:49 UTC
tez git commit: TEZ-3600. Fix flaky test: TestTokenCache. Contributed
by Harish Jaiprakash
Repository: tez
Updated Branches:
refs/heads/master 403a85d76 -> a77d22dd8
TEZ-3600. Fix flaky test: TestTokenCache. Contributed by Harish Jaiprakash
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a77d22dd
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a77d22dd
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a77d22dd
Branch: refs/heads/master
Commit: a77d22dd8ce281ffc24b42b634abb972cb552c78
Parents: 403a85d
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 3 13:46:33 2017 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Feb 3 13:46:33 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../main/java/org/apache/tez/common/security/TokenCache.java | 8 +++++---
.../java/org/apache/tez/common/security/TestTokenCache.java | 3 +++
3 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5d04ecb..bf9c9bc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3600. Fix flaky test: TestTokenCache
TEZ-3589. add a unit test for amKeepAlive not being shutdown if an app takes a long time to launch.
TEZ-3417. Reduce sleep time on AM shutdown to reduce test runtimes
TEZ-3582. Exception swallowed in PipelinedSorter causing incorrect results.
http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
index 0ce5844..fc2c07d 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
@@ -80,13 +80,15 @@ public class TokenCache {
static void obtainTokensForFileSystemsInternal(Credentials credentials,
Path[] ps, Configuration conf) throws IOException {
Set<FileSystem> fsSet = new HashSet<FileSystem>();
+ boolean limitExceeded = false;
for(Path p: ps) {
FileSystem fs = p.getFileSystem(conf);
- if (fsSet.size() == MAX_FS_OBJECTS) {
+ if (!limitExceeded && fsSet.size() == MAX_FS_OBJECTS) {
LOG.warn("No of FileSystem objects exceeds {}, updating tokens for all paths. This can" +
- " happen when fs.<scheme>.impl.disable.cache is set to true.");
+ " happen when fs.<scheme>.impl.disable.cache is set to true.", MAX_FS_OBJECTS);
+ limitExceeded = true;
}
- if (fsSet.size() >= MAX_FS_OBJECTS) {
+ if (limitExceeded) {
// Too many fs objects are being created, most likely the cache is disabled. Prevent an
// OOM and just directly invoke instead of adding to the set.
obtainTokensForFileSystemsInternal(fs, credentials, conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
index 6fc6daa..59488b6 100644
--- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
+++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
@@ -54,6 +55,8 @@ public class TestTokenCache {
public static void setup() throws Exception {
conf = new Configuration();
conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+ conf.setSocketAddr(YarnConfiguration.RM_ADDRESS,
+ InetSocketAddress.createUnresolved("127.0.0.1", 8032));
renewer = Master.getMasterPrincipal(conf);
}