You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/07/20 09:30:14 UTC
incubator-impala git commit: IMPALA-3779: Disable cache pool reader
thread when HDFS isn't running
Repository: incubator-impala
Updated Branches:
refs/heads/master bc8c55afc -> 947305741
IMPALA-3779: Disable cache pool reader thread when HDFS isn't running
When running Impala on S3 (without hdfs started) the logs every
minute produced an error which detailed that the HDFS cache pool
couldn't be read from. This change disables the cache pool reader
thread if HDFS is not the default filesystem.
Change-Id: I2ff3769fff9099aed88a6a587aa721b3487216bd
Reviewed-on: http://gerrit.cloudera.org:8080/3577
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/94730574
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/94730574
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/94730574
Branch: refs/heads/master
Commit: 94730574192d905c8c4b44143f45335cdddb7071
Parents: bc8c55a
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Wed Jul 6 11:32:05 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Jul 20 02:30:09 2016 -0700
----------------------------------------------------------------------
.../cloudera/impala/catalog/CatalogServiceCatalog.java | 12 +++++++++++-
.../java/com/cloudera/impala/common/FileSystemUtil.java | 7 ++++++-
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/94730574/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
index f8ae864..dddc545 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
@@ -14,6 +14,7 @@
package com.cloudera.impala.catalog;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
@@ -160,7 +161,16 @@ public class CatalogServiceCatalog extends Catalog {
catalogServiceId_ = catalogServiceId;
tableLoadingMgr_ = new TableLoadingMgr(this, numLoadingThreads);
loadInBackground_ = loadInBackground;
- cachePoolReader_.scheduleAtFixedRate(new CachePoolReader(), 0, 1, TimeUnit.MINUTES);
+ try {
+ // We want only 'true' HDFS filesystems to poll the HDFS cache (i.e not S3,
+ // local, etc.)
+ if (FileSystemUtil.getDefaultFileSystem() instanceof DistributedFileSystem) {
+ cachePoolReader_.scheduleAtFixedRate(
+ new CachePoolReader(), 0, 1, TimeUnit.MINUTES);
+ }
+ } catch (IOException e) {
+ LOG.error("Couldn't identify the default FS. Cache Pool reader will be disabled.");
+ }
if (sentryConfig != null) {
sentryProxy_ = new SentryProxy(sentryConfig, this, kerberosPrincipal);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/94730574/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
index 1f3da2e..e22ba15 100644
--- a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java
@@ -323,9 +323,14 @@ public class FileSystemUtil {
return isDistributedFileSystem(path.getFileSystem(CONF));
}
- public static DistributedFileSystem getDistributedFileSystem() throws IOException {
+ public static FileSystem getDefaultFileSystem() throws IOException {
Path path = new Path(FileSystem.getDefaultUri(CONF));
FileSystem fs = path.getFileSystem(CONF);
+ return fs;
+ }
+
+ public static DistributedFileSystem getDistributedFileSystem() throws IOException {
+ FileSystem fs = getDefaultFileSystem();
Preconditions.checkState(fs instanceof DistributedFileSystem);
return (DistributedFileSystem) fs;
}