You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/01 01:09:15 UTC
[33/44] hive git commit: HIVE-14824. Separate fstype from cluster
type in QTestUtil. (Siddharth Seth, reviewed by Prasanth Jayachandran)
HIVE-14824. Separate fstype from cluster type in QTestUtil. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf72a737
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf72a737
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf72a737
Branch: refs/heads/hive-14535
Commit: cf72a73708b00b2e585d101258d95eb9cbd3791f
Parents: e2bd513
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Sep 28 13:40:04 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Sep 28 13:40:04 2016 -0700
----------------------------------------------------------------------
.../hive/cli/control/AbstractCliConfig.java | 10 +++
.../hadoop/hive/cli/control/CliConfigs.java | 12 ++-
.../hadoop/hive/cli/control/CoreCliDriver.java | 3 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 84 +++++++++++---------
4 files changed, 69 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index efbd465..03d4075 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.hive.ql.QTestUtil.FsType;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
@@ -63,6 +64,7 @@ public abstract class AbstractCliConfig {
private String initScript;
private String hiveConfDir;
private MiniClusterType clusterType;
+ private FsType fsType;
// FIXME: null value is treated differently on the other end..when those filter will be
// moved...this may change
@@ -380,6 +382,14 @@ public abstract class AbstractCliConfig {
}
}
+ protected FsType getFsType() {
+ return this.fsType;
+ }
+
+ protected void setFsType(FsType fsType) {
+ this.fsType = fsType;
+ }
+
private String getSysPropValue(String propName) {
String propValue = System.getProperty(propName);
if (propValue == null || propValue.trim().length() == 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 0068b95..ca72282 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
+import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.ql.parse.CoreParseNegative;
@@ -171,8 +172,15 @@ public class CliConfigs {
setInitScript("q_test_init_for_encryption.sql");
setCleanupScript("q_test_cleanup_for_encryption.sql");
- setHiveConfDir("data/conf");
- setClusterType(MiniClusterType.encrypted);
+
+ setClusterType(MiniClusterType.mr);
+ setFsType(QTestUtil.FsType.encrypted_hdfs);
+ if (getClusterType() == MiniClusterType.tez) {
+ setHiveConfDir("data/conf/tez");
+ } else {
+ setHiveConfDir("data/conf");
+ }
+
} catch (Exception e) {
throw new RuntimeException("can't construct cliconfig", e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index db58f1d..d83ff45 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -61,7 +61,8 @@ public class CoreCliDriver extends CliAdapter {
@Override
public QTestUtil invokeInternal() throws Exception {
return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, false,
+ cliConfig.getFsType());
}
}.invoke("QtestUtil instance created", LOG, true);
http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 0dfd727..909d7f6 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -51,7 +51,6 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -86,9 +85,7 @@ import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapItUtils;
-import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
@@ -155,7 +152,6 @@ public class QTestUtil {
private String testWarehouse;
private final String testFiles;
- private final boolean useLocalFs;
private final boolean localMode;
protected final String outDir;
protected final String logDir;
@@ -170,6 +166,7 @@ public class QTestUtil {
private static final String SORT_SUFFIX = ".sorted";
private final HashSet<String> srcTables;
private final MiniClusterType clusterType;
+ private final FsType fsType;
private ParseDriver pd;
protected Hive db;
protected QueryState queryState;
@@ -414,14 +411,32 @@ public class QTestUtil {
}
}
+
+ public enum FsType {
+ local,
+ hdfs,
+ encrypted_hdfs,
+ }
+
public enum MiniClusterType {
- mr,
- tez,
- spark,
- encrypted,
- miniSparkOnYarn,
- llap,
- none;
+
+ mr(FsType.hdfs),
+ tez(FsType.hdfs),
+ spark(FsType.local),
+ miniSparkOnYarn(FsType.hdfs),
+ llap(FsType.hdfs),
+ none(FsType.local);
+
+
+ private final FsType defaultFsType;
+
+ MiniClusterType(FsType defaultFsType) {
+ this.defaultFsType = defaultFsType;
+ }
+
+ public FsType getDefaultFsType() {
+ return defaultFsType;
+ }
public static MiniClusterType valueForString(String type) {
if (type.equals("miniMR")) {
@@ -430,8 +445,6 @@ public class QTestUtil {
return tez;
} else if (type.equals("spark")) {
return spark;
- } else if (type.equals("encrypted")) {
- return encrypted;
} else if (type.equals("miniSparkOnYarn")) {
return miniSparkOnYarn;
} else if (type.equals("llap")) {
@@ -442,6 +455,7 @@ public class QTestUtil {
}
}
+
private String getKeyProviderURI() {
// Use the target directory if it is not specified
String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT;
@@ -473,19 +487,24 @@ public class QTestUtil {
String confDir, String hadoopVer, String initScript, String cleanupScript,
boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
- useHBaseMetastore, withLlapIo, false, false);
+ useHBaseMetastore, withLlapIo, false, null);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript,
- boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, boolean useLocalFs)
+ boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, FsType fsType)
throws Exception {
LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," +
" hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," +
- " localMode={}, useLocalFs={}"
+ " localMode={}, fsType={}"
, outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
- useHBaseMetastore, withLlapIo, localMode, useLocalFs);
- this.useLocalFs = useLocalFs;
+ useHBaseMetastore, withLlapIo, localMode, fsType);
+ Preconditions.checkNotNull(clusterType, "ClusterType cannot be null");
+ if (fsType != null) {
+ this.fsType = fsType;
+ } else {
+ this.fsType = clusterType.getDefaultFsType();
+ }
this.localMode = localMode;
this.outDir = outDir;
this.logDir = logDir;
@@ -555,16 +574,12 @@ public class QTestUtil {
private void setupFileSystem(HadoopShims shims) throws IOException {
- if (useLocalFs) {
- Preconditions
- .checkState(clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap,
- "useLocalFs can currently only be set for tez or llap");
- }
-
- if (clusterType != MiniClusterType.none && clusterType != MiniClusterType.spark) {
+ if (fsType == FsType.local) {
+ fs = FileSystem.getLocal(conf);
+ } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) {
int numDataNodes = 4;
- if (clusterType == MiniClusterType.encrypted) {
+ if (fsType == FsType.encrypted_hdfs) {
// Set the security key provider so that the MiniDFS cluster is initialized
// with encryption
conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI());
@@ -578,16 +593,11 @@ public class QTestUtil {
LOG.info("key provider is initialized");
} else {
- if (!useLocalFs) {
- dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
- fs = dfs.getFileSystem();
- } else {
- fs = FileSystem.getLocal(conf);
- }
+ dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
+ fs = dfs.getFileSystem();
}
} else {
- // Setup local file system
- fs = FileSystem.getLocal(conf);
+ throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]");
}
}
@@ -621,7 +631,7 @@ public class QTestUtil {
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
- } else if (clusterType == MiniClusterType.mr || clusterType == MiniClusterType.encrypted) {
+ } else if (clusterType == MiniClusterType.mr) {
mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
}
}
@@ -900,7 +910,7 @@ public class QTestUtil {
if(tblObj.isIndexTable()) {
continue;
}
- db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted);
+ db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs);
} else {
// this table is defined in srcTables, drop all indexes on it
List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1);
@@ -1563,7 +1573,7 @@ public class QTestUtil {
boolean partialMaskWasMatched = false;
Matcher matcher;
while (null != (line = in.readLine())) {
- if (clusterType == MiniClusterType.encrypted) {
+ if (fsType == FsType.encrypted_hdfs) {
for (Pattern pattern : partialReservedPlanMask) {
matcher = pattern.matcher(line);
if (matcher.find()) {