You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ha...@apache.org on 2017/04/19 16:03:25 UTC
tez git commit: TEZ-3600. Fix flaky test: TestTokenCache. Contributed
by Harish Jaiprakash
Repository: tez
Updated Branches:
refs/heads/branch-0.8 bf3666b14 -> ef5904f60
TEZ-3600. Fix flaky test: TestTokenCache. Contributed by Harish Jaiprakash
(cherry picked from commit a77d22dd8ce281ffc24b42b634abb972cb552c78)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ef5904f6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ef5904f6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ef5904f6
Branch: refs/heads/branch-0.8
Commit: ef5904f60fd656fc95bb691c3493804bf1d7330d
Parents: bf3666b
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Feb 3 13:46:33 2017 -0800
Committer: Harish JP <ha...@gmail.com>
Committed: Wed Apr 19 21:32:35 2017 +0530
----------------------------------------------------------------------
.../main/java/org/apache/tez/common/security/TokenCache.java | 8 +++++---
.../java/org/apache/tez/common/security/TestTokenCache.java | 3 +++
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ef5904f6/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
index 0ce5844..fc2c07d 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
@@ -80,13 +80,15 @@ public class TokenCache {
static void obtainTokensForFileSystemsInternal(Credentials credentials,
Path[] ps, Configuration conf) throws IOException {
Set<FileSystem> fsSet = new HashSet<FileSystem>();
+ boolean limitExceeded = false;
for(Path p: ps) {
FileSystem fs = p.getFileSystem(conf);
- if (fsSet.size() == MAX_FS_OBJECTS) {
+ if (!limitExceeded && fsSet.size() == MAX_FS_OBJECTS) {
LOG.warn("No of FileSystem objects exceeds {}, updating tokens for all paths. This can" +
- " happen when fs.<scheme>.impl.disable.cache is set to true.");
+ " happen when fs.<scheme>.impl.disable.cache is set to true.", MAX_FS_OBJECTS);
+ limitExceeded = true;
}
- if (fsSet.size() >= MAX_FS_OBJECTS) {
+ if (limitExceeded) {
// Too many fs objects are being created, most likely the cache is disabled. Prevent an
// OOM and just directly invoke instead of adding to the set.
obtainTokensForFileSystemsInternal(fs, credentials, conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/ef5904f6/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
index 6fc6daa..59488b6 100644
--- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
+++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
@@ -54,6 +55,8 @@ public class TestTokenCache {
public static void setup() throws Exception {
conf = new Configuration();
conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+ conf.setSocketAddr(YarnConfiguration.RM_ADDRESS,
+ InetSocketAddress.createUnresolved("127.0.0.1", 8032));
renewer = Master.getMasterPrincipal(conf);
}