You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2017/03/01 22:48:45 UTC
[2/2] hive git commit: HIVE-14459: TestBeeLineDriver - migration and
re-enable (Peter Vary via Zoltan Haindrich reviewed by Vihang Karajgaonkar)
HIVE-14459: TestBeeLineDriver - migration and re-enable (Peter Vary via Zoltan Haindrich reviewed by Vihang Karajgaonkar)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba8de307
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba8de307
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba8de307
Branch: refs/heads/master
Commit: ba8de3077673e3a9486c8a9d1625b9640936eedc
Parents: 92aaed2
Author: Peter Vary <pv...@cloudera.com>
Authored: Wed Mar 1 23:21:34 2017 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Wed Mar 1 23:47:35 2017 +0100
----------------------------------------------------------------------
.../apache/hive/beeline/util/QFileClient.java | 72 ++-
.../hive/jdbc/miniHS2/AbstractHiveService.java | 159 ------
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 556 -------------------
.../hive/cli/DisabledTestBeeLineDriver.java | 62 ---
.../hadoop/hive/cli/TestBeeLineDriver.java | 62 +++
.../test/resources/testconfiguration.properties | 158 +-----
itests/util/pom.xml | 5 +
.../hadoop/hive/cli/control/CliConfigs.java | 14 +-
.../hive/cli/control/CoreBeeLineDriver.java | 95 ++--
.../hive/jdbc/miniHS2/AbstractHiveService.java | 159 ++++++
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 556 +++++++++++++++++++
.../beeline/escape_comments.q.out | 416 ++++++++++++++
12 files changed, 1307 insertions(+), 1007 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/beeline/src/java/org/apache/hive/beeline/util/QFileClient.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/util/QFileClient.java b/beeline/src/java/org/apache/hive/beeline/util/QFileClient.java
index 81f1b0e..d306b7f 100644
--- a/beeline/src/java/org/apache/hive/beeline/util/QFileClient.java
+++ b/beeline/src/java/org/apache/hive/beeline/util/QFileClient.java
@@ -21,12 +21,15 @@ package org.apache.hive.beeline.util;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.util.Shell;
+import org.apache.hive.common.util.StreamPrinter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,6 +52,8 @@ public class QFileClient {
private File expectedDirectory;
private final File scratchDirectory;
private final File warehouseDirectory;
+ private final File initScript;
+ private final File cleanupScript;
private File testDataDirectory;
private File testScriptDirectory;
@@ -73,11 +78,13 @@ public class QFileClient {
public QFileClient(HiveConf hiveConf, String hiveRootDirectory, String qFileDirectory, String outputDirectory,
- String expectedDirectory) {
+ String expectedDirectory, String initScript, String cleanupScript) {
this.hiveRootDirectory = new File(hiveRootDirectory);
this.qFileDirectory = new File(qFileDirectory);
this.outputDirectory = new File(outputDirectory);
this.expectedDirectory = new File(expectedDirectory);
+ this.initScript = new File(initScript);
+ this.cleanupScript = new File(cleanupScript);
this.scratchDirectory = new File(hiveConf.getVar(ConfVars.SCRATCHDIR));
this.warehouseDirectory = new File(hiveConf.getVar(ConfVars.METASTOREWAREHOUSE));
}
@@ -110,6 +117,9 @@ public class QFileClient {
String timePattern = "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) "
+ "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) "
+ "\\d{2} \\d{2}:\\d{2}:\\d{2} \\w+ 20\\d{2}";
+ // Pattern to remove the timestamp and other infrastructural info from the out file
+ String logPattern = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d*\\s+\\S+\\s+\\[" +
+ ".*\\]\\s+\\S+:\\s+";
String unixTimePattern = "\\D" + currentTimePrefix + "\\d{6}\\D";
String unixTimeMillisPattern = "\\D" + currentTimePrefix + "\\d{9}\\D";
@@ -119,12 +129,15 @@ public class QFileClient {
+ "|SCR|SEL|STATS|TS|UDTF|UNION)_\\d+\"";
filterSet = new RegexFilterSet()
+ .addFilter(logPattern,"")
+ .addFilter("Getting log thread is interrupted, since query is done!\n","")
.addFilter(scratchDirectory.toString() + "[\\w\\-/]+", "!!{hive.exec.scratchdir}!!")
.addFilter(warehouseDirectory.toString(), "!!{hive.metastore.warehouse.dir}!!")
.addFilter(expectedDirectory.toString(), "!!{expectedDirectory}!!")
.addFilter(outputDirectory.toString(), "!!{outputDirectory}!!")
.addFilter(qFileDirectory.toString(), "!!{qFileDirectory}!!")
.addFilter(hiveRootDirectory.toString(), "!!{hive.root}!!")
+ .addFilter("\\(queryId=[^\\)]*\\)","queryId=(!!{queryId}!!)")
.addFilter("file:/\\w\\S+", "file:/!!ELIDED!!")
.addFilter("pfile:/\\w\\S+", "pfile:/!!ELIDED!!")
.addFilter("hdfs:/\\w\\S+", "hdfs:/!!ELIDED!!")
@@ -134,6 +147,7 @@ public class QFileClient {
.addFilter("(\\D)" + currentTimePrefix + "\\d{9}(\\D)", "$1!!UNIXTIMEMILLIS!!$2")
.addFilter(userName, "!!{user.name}!!")
.addFilter(operatorPattern, "\"$1_!!ELIDED!!\"")
+ .addFilter("Time taken: [0-9\\.]* seconds", "Time taken: !!ELIDED!! seconds")
;
};
@@ -219,7 +233,7 @@ public class QFileClient {
"USE `" + testname + "`;",
"set test.data.dir=" + testDataDirectory + ";",
"set test.script.dir=" + testScriptDirectory + ";",
- "!run " + testScriptDirectory + "/q_test_init.sql",
+ "!run " + testScriptDirectory + "/" + initScript,
});
}
@@ -228,6 +242,7 @@ public class QFileClient {
"!set outputformat table",
"USE default;",
"DROP DATABASE IF EXISTS `" + testname + "` CASCADE;",
+ "!run " + testScriptDirectory + "/" + cleanupScript,
});
}
@@ -295,12 +310,61 @@ public class QFileClient {
return expectedFile.exists();
}
- public boolean compareResults() throws IOException {
+ public boolean compareResults() throws IOException, InterruptedException {
if (!expectedFile.exists()) {
LOG.error("Expected results file does not exist: " + expectedFile);
return false;
}
- return FileUtils.contentEquals(expectedFile, outputFile);
+ return executeDiff();
+ }
+
+ private boolean executeDiff() throws IOException, InterruptedException {
+ ArrayList<String> diffCommandArgs = new ArrayList<String>();
+ diffCommandArgs.add("diff");
+
+ // Text file comparison
+ diffCommandArgs.add("-a");
+
+ if (Shell.WINDOWS) {
+ // Ignore changes in the amount of white space
+ diffCommandArgs.add("-b");
+
+ // Files created on Windows machines have different line endings
+ // than files created on Unix/Linux. Windows uses carriage return and line feed
+ // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n").
+ // Also StringBuilder.toString(), Stream to String conversions adds extra
+ // spaces at the end of the line.
+ diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input
+ diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
+ }
+
+ // Add files to compare to the arguments list
+ diffCommandArgs.add(getQuotedString(expectedFile));
+ diffCommandArgs.add(getQuotedString(outputFile));
+
+ System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs,
+ ' '));
+ Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray(
+ new String[diffCommandArgs.size()]));
+
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out);
+
+ outPrinter.start();
+ errPrinter.start();
+
+ int result = executor.waitFor();
+
+ outPrinter.join();
+ errPrinter.join();
+
+ executor.waitFor();
+
+ return (result == 0);
+ }
+
+ private static String getQuotedString(File file) {
+ return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath();
}
public void overwriteResults() {
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
deleted file mode 100644
index 98c5d01..0000000
--- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.jdbc.miniHS2;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-
-/***
- * Base class for Hive service
- * AbstractHiveService.
- *
- */
-public abstract class AbstractHiveService {
- private HiveConf hiveConf = null;
- private String hostname;
- private int binaryPort;
- private int httpPort;
- private boolean startedHiveService = false;
- private List<String> addedProperties = new ArrayList<String>();
-
- public AbstractHiveService(HiveConf hiveConf, String hostname, int binaryPort, int httpPort) {
- this.hiveConf = hiveConf;
- this.hostname = hostname;
- this.binaryPort = binaryPort;
- this.httpPort = httpPort;
- }
-
- /**
- * Get Hive conf
- * @return
- */
- public HiveConf getHiveConf() {
- return hiveConf;
- }
-
- /**
- * Get config property
- * @param propertyKey
- * @return
- */
- public String getConfProperty(String propertyKey) {
- return hiveConf.get(propertyKey);
- }
-
- /**
- * Set config property
- * @param propertyKey
- * @param propertyValue
- */
- public void setConfProperty(String propertyKey, String propertyValue) {
- System.setProperty(propertyKey, propertyValue);
- hiveConf.set(propertyKey, propertyValue);
- addedProperties.add(propertyKey);
- }
-
- /**
- * Create system properties set by this server instance. This ensures that
- * the changes made by current test are not impacting subsequent tests.
- */
- public void clearProperties() {
- for (String propKey : addedProperties ) {
- System.clearProperty(propKey);
- }
- }
-
- /**
- * Retrieve warehouse directory
- * @return
- */
- public Path getWareHouseDir() {
- return new Path(hiveConf.getVar(ConfVars.METASTOREWAREHOUSE));
- }
-
- public void setWareHouseDir(String wareHouseURI) {
- verifyNotStarted();
- System.setProperty(ConfVars.METASTOREWAREHOUSE.varname, wareHouseURI);
- hiveConf.setVar(ConfVars.METASTOREWAREHOUSE, wareHouseURI);
- }
-
- /**
- * Set service host
- * @param hostName
- */
- public void setHost(String hostName) {
- this.hostname = hostName;
- }
-
- // get service host
- public String getHost() {
- return hostname;
- }
-
- /**
- * Set binary service port #
- * @param portNum
- */
- public void setBinaryPort(int portNum) {
- this.binaryPort = portNum;
- }
-
- /**
- * Set http service port #
- * @param portNum
- */
- public void setHttpPort(int portNum) {
- this.httpPort = portNum;
- }
-
- // Get binary service port #
- public int getBinaryPort() {
- return binaryPort;
- }
-
- // Get http service port #
- public int getHttpPort() {
- return httpPort;
- }
-
- public boolean isStarted() {
- return startedHiveService;
- }
-
- public void setStarted(boolean hiveServiceStatus) {
- this.startedHiveService = hiveServiceStatus;
- }
-
- protected void verifyStarted() {
- if (!isStarted()) {
- throw new IllegalStateException("HiveServer2 is not running");
- }
- }
-
- protected void verifyNotStarted() {
- if (isStarted()) {
- throw new IllegalStateException("HiveServer2 already running");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
deleted file mode 100644
index 71f9640..0000000
--- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ /dev/null
@@ -1,556 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.jdbc.miniHS2;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapItUtils;
-import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
-import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
-import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.Service;
-import org.apache.hive.service.cli.CLIServiceClient;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
-import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
-import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
-import org.apache.hive.service.server.HiveServer2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MiniHS2 extends AbstractHiveService {
-
- private static final Logger LOG = LoggerFactory.getLogger(MiniHS2.class);
-
- public static final String HS2_BINARY_MODE = "binary";
- public static final String HS2_HTTP_MODE = "http";
- private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
- private static final FsPermission FULL_PERM = new FsPermission((short)00777);
- private static final FsPermission WRITE_ALL_PERM = new FsPermission((short)00733);
- private static final String tmpDir = System.getProperty("test.tmp.dir");
- private HiveServer2 hiveServer2 = null;
- private final File baseDir;
- private final Path baseFsDir;
- private MiniMrShim mr;
- private MiniDFSShim dfs;
- private MiniLlapCluster llapCluster = null;
- private final FileSystem localFS;
- private boolean useMiniKdc = false;
- private final String serverPrincipal;
- private final boolean isMetastoreRemote;
- private final boolean cleanupLocalDirOnStartup;
- private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
-
- public enum MiniClusterType {
- MR,
- TEZ,
- LLAP,
- LOCALFS_ONLY;
- }
-
- public static class Builder {
- private HiveConf hiveConf = new HiveConf();
- private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
- private boolean useMiniKdc = false;
- private String serverPrincipal;
- private String serverKeytab;
- private boolean isHTTPTransMode = false;
- private boolean isMetastoreRemote;
- private boolean usePortsFromConf = false;
- private String authType = "KERBEROS";
- private boolean isHA = false;
- private boolean cleanupLocalDirOnStartup = true;
-
- public Builder() {
- }
-
- public Builder withMiniMR() {
- this.miniClusterType = MiniClusterType.MR;
- return this;
- }
-
- public Builder withMiniKdc(String serverPrincipal, String serverKeytab) {
- this.useMiniKdc = true;
- this.serverPrincipal = serverPrincipal;
- this.serverKeytab = serverKeytab;
- return this;
- }
-
- public Builder withAuthenticationType(String authType) {
- this.authType = authType;
- return this;
- }
-
- public Builder withRemoteMetastore() {
- this.isMetastoreRemote = true;
- return this;
- }
-
- public Builder withConf(HiveConf hiveConf) {
- this.hiveConf = hiveConf;
- return this;
- }
-
- public Builder withHA() {
- this.isHA = true;
- return this;
- }
-
- /**
- * Start HS2 with HTTP transport mode, default is binary mode
- * @return this Builder
- */
- public Builder withHTTPTransport(){
- this.isHTTPTransMode = true;
- return this;
- }
-
- public Builder cleanupLocalDirOnStartup(boolean val) {
- this.cleanupLocalDirOnStartup = val;
- return this;
- }
-
- public MiniHS2 build() throws Exception {
- if (miniClusterType == MiniClusterType.MR && useMiniKdc) {
- throw new IOException("Can't create secure miniMr ... yet");
- }
- if (isHTTPTransMode) {
- hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE);
- } else {
- hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE);
- }
- return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab,
- isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup);
- }
- }
-
- public MiniMrShim getMr() {
- return mr;
- }
-
- public void setMr(MiniMrShim mr) {
- this.mr = mr;
- }
-
- public MiniDFSShim getDfs() {
- return dfs;
- }
-
- public void setDfs(MiniDFSShim dfs) {
- this.dfs = dfs;
- }
-
- public FileSystem getLocalFS() {
- return localFS;
- }
-
- public MiniClusterType getMiniClusterType() {
- return miniClusterType;
- }
-
- public void setMiniClusterType(MiniClusterType miniClusterType) {
- this.miniClusterType = miniClusterType;
- }
-
- public boolean isUseMiniKdc() {
- return useMiniKdc;
- }
-
- private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc,
- String serverPrincipal, String serverKeytab, boolean isMetastoreRemote,
- boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception {
- // Always use localhost for hostname as some tests like SSL CN validation ones
- // are tied to localhost being present in the certificate name
- super(
- hiveConf,
- "localhost",
- (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT) : MetaStoreUtils
- .findFreePort()),
- (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreUtils
- .findFreePort()));
- hiveConf.setLongVar(ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS, 3l);
- hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS, 10,
- TimeUnit.SECONDS);
- this.miniClusterType = miniClusterType;
- this.useMiniKdc = useMiniKdc;
- this.serverPrincipal = serverPrincipal;
- this.isMetastoreRemote = isMetastoreRemote;
- this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup;
- baseDir = getBaseDir();
- localFS = FileSystem.getLocal(hiveConf);
- FileSystem fs;
-
- if (miniClusterType != MiniClusterType.LOCALFS_ONLY) {
- // Initialize dfs
- dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA);
- fs = dfs.getFileSystem();
- String uriString = fs.getUri().toString();
-
- // Initialize the execution engine based on cluster type
- switch (miniClusterType) {
- case TEZ:
- // Change the engine to tez
- hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
- // TODO: This should be making use of confDir to load configs setup for Tez, etc.
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, false);
- break;
- case LLAP:
- if (usePortsFromConf) {
- hiveConf.setBoolean("minillap.usePortsFromConf", true);
- }
- llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null);
-
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, true);
- break;
- case MR:
- mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1);
- break;
- default:
- throw new IllegalArgumentException("Unsupported cluster type " + mr);
- }
- // store the config in system properties
- mr.setupConfiguration(getHiveConf());
- baseFsDir = new Path(new Path(fs.getUri()), "/base");
- } else {
- // This is FS only mode, just initialize the dfs root directory.
- fs = FileSystem.getLocal(hiveConf);
- baseFsDir = new Path("file://" + baseDir.toURI().getPath());
-
- if (cleanupLocalDirOnStartup) {
- // Cleanup baseFsDir since it can be shared across tests.
- LOG.info("Attempting to cleanup baseFsDir: {} while setting up MiniHS2", baseDir);
- Preconditions.checkState(baseFsDir.depth() >= 3); // Avoid "/tmp", directories closer to "/"
- fs.delete(baseFsDir, true);
- }
- }
- if (useMiniKdc) {
- hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal);
- hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab);
- hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType);
- }
- String metaStoreURL =
- "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator
- + "test_metastore;create=true";
-
- fs.mkdirs(baseFsDir);
- Path wareHouseDir = new Path(baseFsDir, "warehouse");
- // Create warehouse with 777, so that user impersonation has no issues.
- FileSystem.mkdirs(fs, wareHouseDir, FULL_PERM);
-
- fs.mkdirs(wareHouseDir);
- setWareHouseDir(wareHouseDir.toString());
- System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL);
- hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, metaStoreURL);
- if (!usePortsFromConf) {
- // reassign a new port, just in case if one of the MR services grabbed the last one
- setBinaryPort(MetaStoreUtils.findFreePort());
- }
- hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, getHost());
- hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort());
- hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort());
-
- Path scratchDir = new Path(baseFsDir, "scratch");
- // Create root scratchdir with write all, so that user impersonation has no issues.
- Utilities.createDirsWithPermission(hiveConf, scratchDir, WRITE_ALL_PERM, true);
- System.setProperty(HiveConf.ConfVars.SCRATCHDIR.varname, scratchDir.toString());
- hiveConf.setVar(ConfVars.SCRATCHDIR, scratchDir.toString());
-
- String localScratchDir = baseDir.getPath() + File.separator + "scratch";
- System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.varname, localScratchDir);
- hiveConf.setVar(ConfVars.LOCALSCRATCHDIR, localScratchDir);
- }
-
- public MiniHS2(HiveConf hiveConf) throws Exception {
- this(hiveConf, MiniClusterType.LOCALFS_ONLY);
- }
-
- public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception {
- this(hiveConf, clusterType, false);
- }
-
- public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType,
- boolean usePortsFromConf) throws Exception {
- this(hiveConf, clusterType, false, null, null, false, usePortsFromConf,
- "KERBEROS", false, true);
- }
-
- public void start(Map<String, String> confOverlay) throws Exception {
- if (isMetastoreRemote) {
- int metaStorePort = MetaStoreUtils.findFreePort();
- getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
- MetaStoreUtils.startMetaStore(metaStorePort,
- ShimLoader.getHadoopThriftAuthBridge(), getHiveConf());
- }
-
- hiveServer2 = new HiveServer2();
- // Set confOverlay parameters
- for (Map.Entry<String, String> entry : confOverlay.entrySet()) {
- setConfProperty(entry.getKey(), entry.getValue());
- }
- hiveServer2.init(getHiveConf());
- hiveServer2.start();
- waitForStartup();
- setStarted(true);
- }
-
- public void stop() {
- verifyStarted();
- // Currently there is no way to stop the MetaStore service. It will be stopped when the
- // test JVM exits. This is how other tests are also using MetaStore server.
-
- hiveServer2.stop();
- setStarted(false);
- try {
- if (llapCluster != null) {
- llapCluster.stop();
- }
- if (mr != null) {
- mr.shutdown();
- mr = null;
- }
- if (dfs != null) {
- dfs.shutdown();
- dfs = null;
- }
- } catch (IOException e) {
- // Ignore errors cleaning up miniMR
- }
- }
-
- public void cleanup() {
- FileUtils.deleteQuietly(baseDir);
- }
-
-
- public CLIServiceClient getServiceClient() {
- verifyStarted();
- return getServiceClientInternal();
- }
-
- public HiveConf getServerConf() {
- if (hiveServer2 != null) {
- return hiveServer2.getHiveConf();
- }
- return null;
- }
-
- public CLIServiceClient getServiceClientInternal() {
- for (Service service : hiveServer2.getServices()) {
- if (service instanceof ThriftBinaryCLIService) {
- return new ThriftCLIServiceClient((ThriftBinaryCLIService) service);
- }
- if (service instanceof ThriftHttpCLIService) {
- return new ThriftCLIServiceClient((ThriftHttpCLIService) service);
- }
- }
- throw new IllegalStateException("HiveServer2 not running Thrift service");
- }
-
- /**
- * return connection URL for this server instance
- * @return
- * @throws Exception
- */
- public String getJdbcURL() throws Exception {
- return getJdbcURL("default");
- }
-
- /**
- * return connection URL for this server instance
- * @param dbName - DB name to be included in the URL
- * @return
- * @throws Exception
- */
- public String getJdbcURL(String dbName) throws Exception {
- return getJdbcURL(dbName, "");
- }
-
- /**
- * return connection URL for this server instance
- * @param dbName - DB name to be included in the URL
- * @param sessionConfExt - Addional string to be appended to sessionConf part of url
- * @return
- * @throws Exception
- */
- public String getJdbcURL(String dbName, String sessionConfExt) throws Exception {
- return getJdbcURL(dbName, sessionConfExt, "");
- }
-
- /**
- * return connection URL for this server instance
- * @param dbName - DB name to be included in the URL
- * @param sessionConfExt - Addional string to be appended to sessionConf part of url
- * @param hiveConfExt - Additional string to be appended to HiveConf part of url (excluding the ?)
- * @return
- * @throws Exception
- */
- public String getJdbcURL(String dbName, String sessionConfExt, String hiveConfExt)
- throws Exception {
- sessionConfExt = (sessionConfExt == null ? "" : sessionConfExt);
- hiveConfExt = (hiveConfExt == null ? "" : hiveConfExt);
- // Strip the leading ";" if provided
- // (this is the assumption with which we're going to start configuring sessionConfExt)
- if (sessionConfExt.startsWith(";")) {
- sessionConfExt = sessionConfExt.substring(1);
- }
- if (isUseMiniKdc()) {
- sessionConfExt = "principal=" + serverPrincipal + ";" + sessionConfExt;
- }
- if (isHttpTransportMode()) {
- sessionConfExt = "transportMode=http;httpPath=cliservice" + ";" + sessionConfExt;
- }
- String baseJdbcURL;
- if (isDynamicServiceDiscovery()) {
- sessionConfExt =
- "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace="
- + getServerConf().getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE) + ";"
- + sessionConfExt;
- baseJdbcURL = getZKBaseJdbcURL();
- } else {
- baseJdbcURL = getBaseJdbcURL();
- }
-
- baseJdbcURL = baseJdbcURL + dbName;
- if (!sessionConfExt.isEmpty()) {
- baseJdbcURL = baseJdbcURL + ";" + sessionConfExt;
- }
- if ((hiveConfExt != null) && (!hiveConfExt.trim().isEmpty())) {
- baseJdbcURL = baseJdbcURL + "?" + hiveConfExt;
- }
- return baseJdbcURL;
- }
-
- /**
- * Build base JDBC URL
- * @return
- */
- public String getBaseJdbcURL() {
- if(isHttpTransportMode()) {
- return "jdbc:hive2://" + getHost() + ":" + getHttpPort() + "/";
- }
- else {
- return "jdbc:hive2://" + getHost() + ":" + getBinaryPort() + "/";
- }
- }
-
- /**
- * Build zk base JDBC URL
- * @return
- */
- private String getZKBaseJdbcURL() throws Exception {
- HiveConf hiveConf = getServerConf();
- if (hiveConf != null) {
- String zkEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
- return "jdbc:hive2://" + zkEnsemble + "/";
- }
- throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
- }
-
- private boolean isHttpTransportMode() {
- String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname);
- return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE));
- }
-
- private boolean isDynamicServiceDiscovery() throws Exception {
- HiveConf hiveConf = getServerConf();
- if (hiveConf == null) {
- throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
- }
- if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
- return true;
- }
- return false;
- }
-
- public static String getJdbcDriverName() {
- return driverName;
- }
-
- public MiniMrShim getMR() {
- return mr;
- }
-
- public MiniDFSShim getDFS() {
- return dfs;
- }
-
- private void waitForStartup() throws Exception {
- int waitTime = 0;
- long startupTimeout = 1000L * 1000L;
- CLIServiceClient hs2Client = getServiceClientInternal();
- SessionHandle sessionHandle = null;
- do {
- Thread.sleep(500L);
- waitTime += 500L;
- if (waitTime > startupTimeout) {
- throw new TimeoutException("Couldn't access new HiveServer2: " + getJdbcURL());
- }
- try {
- Map <String, String> sessionConf = new HashMap<String, String>();
- /**
- if (isUseMiniKdc()) {
- getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal());
- sessionConf.put("principal", serverPrincipal);
- }
- */
- sessionHandle = hs2Client.openSession("foo", "bar", sessionConf);
- } catch (Exception e) {
- // service not started yet
- continue;
- }
- hs2Client.closeSession(sessionHandle);
- break;
- } while (true);
- }
-
- public Service.STATE getState() {
- return hiveServer2.getServiceState();
- }
-
- static File getBaseDir() {
- File baseDir = new File(tmpDir + "/local_base");
- return baseDir;
- }
-
- public static void cleanupLocalDir() throws IOException {
- File baseDir = getBaseDir();
- try {
- org.apache.hadoop.hive.common.FileUtils.deleteDirectory(baseDir);
- } catch (FileNotFoundException e) {
- // Ignore. Safe if it does not exist.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/DisabledTestBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/DisabledTestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/DisabledTestBeeLineDriver.java
deleted file mode 100644
index cb276e6..0000000
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/DisabledTestBeeLineDriver.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class DisabledTestBeeLineDriver {
-
- static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public DisabledTestBeeLineDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
new file mode 100644
index 0000000..24eeb9d
--- /dev/null
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.hadoop.hive.cli.control.CliAdapter;
+import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestBeeLineDriver {
+
+ static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter();
+
+ @Parameters(name = "{0}")
+ public static List<Object[]> getParameters() throws Exception {
+ return adapter.getParameters();
+ }
+
+ @ClassRule
+ public static TestRule cliClassRule = adapter.buildClassRule();
+
+ @Rule
+ public TestRule cliTestRule = adapter.buildTestRule();
+
+ private String name;
+ private File qfile;
+
+ public TestBeeLineDriver(String name, File qfile) {
+ this.name = name;
+ this.qfile = qfile;
+ }
+
+ @Test
+ public void testCliDriver() throws Exception {
+ adapter.runTest(name, qfile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 9c6a069..b01ebd8 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -722,163 +722,7 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_with_trash.q \
encryption_ctas.q
-beeline.positive.exclude=add_part_exist.q,\
- alter1.q,\
- alter2.q,\
- alter4.q,\
- alter5.q,\
- alter_rename_partition.q,\
- alter_rename_partition_authorization.q,\
- archive.q,\
- archive_corrupt.q,\
- archive_mr_1806.q,\
- archive_multi.q,\
- archive_multi_mr_1806.q,\
- authorization_1.q,\
- authorization_2.q,\
- authorization_4.q,\
- authorization_5.q,\
- authorization_6.q,\
- authorization_7.q,\
- ba_table1.q,\
- ba_table2.q,\
- ba_table3.q,\
- ba_table_udfs.q,\
- binary_table_bincolserde.q,\
- binary_table_colserde.q,\
- cluster.q,\
- columnarserde_create_shortcut.q,\
- combine2.q,\
- constant_prop.q,\
- create_nested_type.q,\
- create_or_replace_view.q,\
- create_struct_table.q,\
- create_union_table.q,\
- database.q,\
- database_location.q,\
- database_properties.q,\
- describe_database_json.q,\
- drop_database_removes_partition_dirs.q,\
- escape1.q,\
- escape2.q,\
- exim_00_nonpart_empty.q,\
- exim_01_nonpart.q,\
- exim_02_00_part_empty.q,\
- exim_02_part.q,\
- exim_03_nonpart_over_compat.q,\
- exim_04_all_part.q,\
- exim_04_evolved_parts.q,\
- exim_05_some_part.q,\
- exim_06_one_part.q,\
- exim_07_all_part_over_nonoverlap.q,\
- exim_08_nonpart_rename.q,\
- exim_09_part_spec_nonoverlap.q,\
- exim_10_external_managed.q,\
- exim_11_managed_external.q,\
- exim_12_external_location.q,\
- exim_13_managed_location.q,\
- exim_14_managed_location_over_existing.q,\
- exim_15_external_part.q,\
- exim_16_part_external.q,\
- exim_17_part_managed.q,\
- exim_18_part_external.q,\
- exim_19_00_part_external_location.q,\
- exim_19_part_external_location.q,\
- exim_20_part_managed_location.q,\
- exim_21_export_authsuccess.q,\
- exim_22_import_exist_authsuccess.q,\
- exim_23_import_part_authsuccess.q,\
- exim_24_import_nonexist_authsuccess.q,\
- global_limit.q,\
- groupby_complex_types.q,\
- groupby_complex_types_multi_single_reducer.q,\
- index_auth.q,\
- index_auto.q,\
- index_auto_empty.q,\
- index_bitmap.q,\
- index_bitmap1.q,\
- index_bitmap2.q,\
- index_bitmap3.q,\
- index_bitmap_auto.q,\
- index_bitmap_rc.q,\
- index_compact.q,\
- index_compact_1.q,\
- index_compact_2.q,\
- index_compact_3.q,\
- index_stale_partitioned.q,\
- init_file.q,\
- input16.q,\
- input16_cc.q,\
- input46.q,\
- input_columnarserde.q,\
- input_dynamicserde.q,\
- input_lazyserde.q,\
- input_testxpath3.q,\
- input_testxpath4.q,\
- insert2_overwrite_partitions.q,\
- insertexternal1.q,\
- join_thrift.q,\
- lateral_view.q,\
- load_binary_data.q,\
- load_exist_part_authsuccess.q,\
- load_nonpart_authsuccess.q,\
- load_part_authsuccess.q,\
- loadpart_err.q,\
- lock1.q,\
- lock2.q,\
- lock3.q,\
- lock4.q,\
- merge_dynamic_partition.q,\
- multi_insert.q,\
- multi_insert_move_tasks_share_dependencies.q,\
- null_column.q,\
- ppd_clusterby.q,\
- query_with_semi.q,\
- rename_column.q,\
- sample6.q,\
- sample_islocalmode_hook.q,\
- set_processor_namespaces.q,\
- show_tables.q,\
- source.q,\
- split_sample.q,\
- str_to_map.q,\
- transform1.q,\
- udaf_collect_set.q,\
- udaf_context_ngrams.q,\
- udaf_histogram_numeric.q,\
- udaf_ngrams.q,\
- udaf_percentile_approx.q,\
- udf_array.q,\
- udf_bitmap_and.q,\
- udf_bitmap_or.q,\
- udf_explode.q,\
- udf_format_number.q,\
- udf_map.q,\
- udf_map_keys.q,\
- udf_map_values.q,\
- udf_mask.q,\
- udf_mask_first_n.q,\
- udf_mask_hash.q,\
- udf_mask_last_n.q,\
- udf_mask_show_first_n.q,\
- udf_mask_show_last_n.q,\
- udf_max.q,\
- udf_min.q,\
- udf_named_struct.q,\
- udf_percentile.q,\
- udf_printf.q,\
- udf_sentences.q,\
- udf_sort_array.q,\
- udf_sort_array_by.q,\
- udf_split.q,\
- udf_struct.q,\
- udf_substr.q,\
- udf_translate.q,\
- udf_union.q,\
- udf_xpath.q,\
- udtf_stack.q,\
- view.q,\
- virtual_column.q
+beeline.positive.include=escape_comments.q
minimr.query.negative.files=cluster_tasklog_retrieval.q,\
file_with_header_footer_negative.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index 6d93dc1..3740af8 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -71,6 +71,11 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-beeline</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-hbase-handler</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index af8ec67..67064b8 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -406,19 +406,17 @@ public class CliConfigs {
public static class BeeLineConfig extends AbstractCliConfig {
public BeeLineConfig() {
- // FIXME: beeline is disabled...
- super(null);
- // super(CoreBeeLineDriver.class);
+ super(CoreBeeLineDriver.class);
try {
setQueryDir("ql/src/test/queries/clientpositive");
- excludesFrom(testConfigProps, "beeline.positive.exclude");
+ includesFrom(testConfigProps, "beeline.positive.include");
- setResultsDir("ql/src/test/results/clientpositive");
+ setResultsDir("ql/src/test/results/clientpositive/beeline");
setLogDir("itests/qtest/target/qfile-results/beelinepositive");
- setInitScript("q_test_init.sql");
- setCleanupScript("q_test_cleanup.sql");
+ setInitScript("q_test_init_src.sql");
+ setCleanupScript("q_test_cleanup_src.sql");
setHiveConfDir("");
setClusterType(MiniClusterType.none);
@@ -434,8 +432,6 @@ public class CliConfigs {
try {
setQueryDir("accumulo-handler/src/test/queries/positive");
- excludesFrom(testConfigProps, "beeline.positive.exclude");
-
setResultsDir("accumulo-handler/src/test/results/positive");
setLogDir("itests/qtest/target/qfile-results/accumulo-handler/positive");
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index e5144e3..aba1fde 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -16,104 +16,87 @@
* limitations under the License.
*/
package org.apache.hadoop.hive.cli.control;
-//beeline is excluded by default
-//AFAIK contains broken tests
-//and produces compile errors...i'll comment out this whole class for now...
-/*
import static org.junit.Assert.fail;
-import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.*;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hive.beeline.util.QFileClient;
-import org.apache.hive.service.server.HiveServer2;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-// HIVE-14444: i've dropped this: @RunWith(ConcurrentTestRunner.class)
+
+import java.util.HashMap;
+
public class CoreBeeLineDriver extends CliAdapter {
private final String hiveRootDirectory = AbstractCliConfig.HIVE_ROOT;
private final String queryDirectory;
private final String logDirectory;
private final String resultsDirectory;
+ private final String initScript;
+ private final String cleanupScript;
private boolean overwrite = false;
- private static String scratchDirectory;
- private static QTestUtil.QTestSetup miniZKCluster = null;
-
- private static HiveServer2 hiveServer2;
+ private MiniHS2 miniHS2;
+// private static QTestUtil.QTestSetup miniZKCluster = null;
public CoreBeeLineDriver(AbstractCliConfig testCliConfig) {
super(testCliConfig);
queryDirectory = testCliConfig.getQueryDirectory();
logDirectory = testCliConfig.getLogDir();
resultsDirectory = testCliConfig.getResultsDir();
+ initScript = testCliConfig.getInitScript();
+ cleanupScript = testCliConfig.getCleanupScript();
}
@Override
@BeforeClass
public void beforeClass() throws Exception {
- HiveConf hiveConf = new HiveConf();
- hiveConf.logVars(System.err);
- System.err.flush();
-
- scratchDirectory = hiveConf.getVar(SCRATCHDIR);
-
String testOutputOverwrite = System.getProperty("test.output.overwrite");
if (testOutputOverwrite != null && "true".equalsIgnoreCase(testOutputOverwrite)) {
overwrite = true;
}
- miniZKCluster = new QTestUtil.QTestSetup();
- miniZKCluster.preTest(hiveConf);
-
- System.setProperty("hive.zookeeper.quorum",
- hiveConf.get("hive.zookeeper.quorum"));
- System.setProperty("hive.zookeeper.client.port",
- hiveConf.get("hive.zookeeper.client.port"));
-
String disableserver = System.getProperty("test.service.disable.server");
if (null != disableserver && disableserver.equalsIgnoreCase("true")) {
- System.err.println("test.service.disable.server=true "
- + "Skipping HiveServer2 initialization!");
+ System.err.println("test.service.disable.server=true Skipping HiveServer2 initialization!");
return;
}
- hiveServer2 = new HiveServer2();
- hiveServer2.init(hiveConf);
- System.err.println("Starting HiveServer2...");
- hiveServer2.start();
- Thread.sleep(5000);
+ HiveConf hiveConf = new HiveConf();
+ // We do not need Zookeeper at the moment
+ hiveConf.set(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname,
+ "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
+
+ // But if we need later we can enable it with this, or create one ourself
+// miniZKCluster = new QTestUtil.QTestSetup();
+// miniZKCluster.preTest(hiveConf);
+
+ hiveConf.logVars(System.err);
+ System.err.flush();
+
+ miniHS2 = new MiniHS2.Builder().withConf(hiveConf).cleanupLocalDirOnStartup(true).build();
+
+ miniHS2.start(new HashMap<String, String>());
}
@Override
@AfterClass
- public void shutdown() {
- try {
- if (hiveServer2 != null) {
- System.err.println("Stopping HiveServer2...");
- hiveServer2.stop();
- }
- } catch (Throwable t) {
- t.printStackTrace();
- }
-
- if (miniZKCluster != null) {
- try {
- miniZKCluster.tearDown();
- } catch (Exception e) {
- e.printStackTrace();
- }
+ public void shutdown() throws Exception {
+ if (miniHS2 != null) {
+ miniHS2.stop();
}
+// if (miniZKCluster != null) {
+// miniZKCluster.tearDown();
+// }
}
public void runTest(String qFileName) throws Exception {
- QFileClient qClient = new QFileClient(new HiveConf(), hiveRootDirectory,
- queryDirectory, logDirectory, resultsDirectory)
+ QFileClient qClient = new QFileClient(miniHS2.getHiveConf(), hiveRootDirectory,
+ queryDirectory, logDirectory, resultsDirectory, initScript, cleanupScript)
.setQFileName(qFileName)
.setUsername("user")
.setPassword("password")
- .setJdbcUrl("jdbc:hive2://localhost:10000")
+ .setJdbcUrl(miniHS2.getJdbcURL())
.setJdbcDriver("org.apache.hive.jdbc.HiveDriver")
.setTestDataDirectory(hiveRootDirectory + "/data/files")
.setTestScriptDirectory(hiveRootDirectory + "/data/scripts");
@@ -150,22 +133,14 @@ public class CoreBeeLineDriver extends CliAdapter {
@Override
public void setUp() {
- // TODO Auto-generated method stub
-
}
@Override
public void tearDown() {
- // TODO Auto-generated method stub
-
}
@Override
public void runTest(String name, String name2, String absolutePath) throws Exception {
runTest(name2);
}
-
}
-
-
-*/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
new file mode 100644
index 0000000..98c5d01
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc.miniHS2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+/***
+ * Base class for Hive service
+ * AbstractHiveService.
+ *
+ */
+public abstract class AbstractHiveService {
+ private HiveConf hiveConf = null;
+ private String hostname;
+ private int binaryPort;
+ private int httpPort;
+ private boolean startedHiveService = false;
+ private List<String> addedProperties = new ArrayList<String>();
+
+ public AbstractHiveService(HiveConf hiveConf, String hostname, int binaryPort, int httpPort) {
+ this.hiveConf = hiveConf;
+ this.hostname = hostname;
+ this.binaryPort = binaryPort;
+ this.httpPort = httpPort;
+ }
+
+ /**
+ * Get Hive conf
+ * @return
+ */
+ public HiveConf getHiveConf() {
+ return hiveConf;
+ }
+
+ /**
+ * Get config property
+ * @param propertyKey
+ * @return
+ */
+ public String getConfProperty(String propertyKey) {
+ return hiveConf.get(propertyKey);
+ }
+
+ /**
+ * Set config property
+ * @param propertyKey
+ * @param propertyValue
+ */
+ public void setConfProperty(String propertyKey, String propertyValue) {
+ System.setProperty(propertyKey, propertyValue);
+ hiveConf.set(propertyKey, propertyValue);
+ addedProperties.add(propertyKey);
+ }
+
+ /**
+ * Create system properties set by this server instance. This ensures that
+ * the changes made by current test are not impacting subsequent tests.
+ */
+ public void clearProperties() {
+ for (String propKey : addedProperties ) {
+ System.clearProperty(propKey);
+ }
+ }
+
+ /**
+ * Retrieve warehouse directory
+ * @return
+ */
+ public Path getWareHouseDir() {
+ return new Path(hiveConf.getVar(ConfVars.METASTOREWAREHOUSE));
+ }
+
+ public void setWareHouseDir(String wareHouseURI) {
+ verifyNotStarted();
+ System.setProperty(ConfVars.METASTOREWAREHOUSE.varname, wareHouseURI);
+ hiveConf.setVar(ConfVars.METASTOREWAREHOUSE, wareHouseURI);
+ }
+
+ /**
+ * Set service host
+ * @param hostName
+ */
+ public void setHost(String hostName) {
+ this.hostname = hostName;
+ }
+
+ // get service host
+ public String getHost() {
+ return hostname;
+ }
+
+ /**
+ * Set binary service port #
+ * @param portNum
+ */
+ public void setBinaryPort(int portNum) {
+ this.binaryPort = portNum;
+ }
+
+ /**
+ * Set http service port #
+ * @param portNum
+ */
+ public void setHttpPort(int portNum) {
+ this.httpPort = portNum;
+ }
+
+ // Get binary service port #
+ public int getBinaryPort() {
+ return binaryPort;
+ }
+
+ // Get http service port #
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ public boolean isStarted() {
+ return startedHiveService;
+ }
+
+ public void setStarted(boolean hiveServiceStatus) {
+ this.startedHiveService = hiveServiceStatus;
+ }
+
+ protected void verifyStarted() {
+ if (!isStarted()) {
+ throw new IllegalStateException("HiveServer2 is not running");
+ }
+ }
+
+ protected void verifyNotStarted() {
+ if (isStarted()) {
+ throw new IllegalStateException("HiveServer2 already running");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8de307/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
new file mode 100644
index 0000000..71f9640
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -0,0 +1,556 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc.miniHS2;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapItUtils;
+import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
+import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
+import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.Service;
+import org.apache.hive.service.cli.CLIServiceClient;
+import org.apache.hive.service.cli.SessionHandle;
+import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
+import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
+import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
+import org.apache.hive.service.server.HiveServer2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MiniHS2 extends AbstractHiveService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MiniHS2.class);
+
+ public static final String HS2_BINARY_MODE = "binary";
+ public static final String HS2_HTTP_MODE = "http";
+ private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
+ private static final FsPermission FULL_PERM = new FsPermission((short)00777);
+ private static final FsPermission WRITE_ALL_PERM = new FsPermission((short)00733);
+ private static final String tmpDir = System.getProperty("test.tmp.dir");
+ private HiveServer2 hiveServer2 = null;
+ private final File baseDir;
+ private final Path baseFsDir;
+ private MiniMrShim mr;
+ private MiniDFSShim dfs;
+ private MiniLlapCluster llapCluster = null;
+ private final FileSystem localFS;
+ private boolean useMiniKdc = false;
+ private final String serverPrincipal;
+ private final boolean isMetastoreRemote;
+ private final boolean cleanupLocalDirOnStartup;
+ private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
+
+ public enum MiniClusterType {
+ MR,
+ TEZ,
+ LLAP,
+ LOCALFS_ONLY;
+ }
+
+ public static class Builder {
+ private HiveConf hiveConf = new HiveConf();
+ private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
+ private boolean useMiniKdc = false;
+ private String serverPrincipal;
+ private String serverKeytab;
+ private boolean isHTTPTransMode = false;
+ private boolean isMetastoreRemote;
+ private boolean usePortsFromConf = false;
+ private String authType = "KERBEROS";
+ private boolean isHA = false;
+ private boolean cleanupLocalDirOnStartup = true;
+
+ public Builder() {
+ }
+
+ public Builder withMiniMR() {
+ this.miniClusterType = MiniClusterType.MR;
+ return this;
+ }
+
+ public Builder withMiniKdc(String serverPrincipal, String serverKeytab) {
+ this.useMiniKdc = true;
+ this.serverPrincipal = serverPrincipal;
+ this.serverKeytab = serverKeytab;
+ return this;
+ }
+
+ public Builder withAuthenticationType(String authType) {
+ this.authType = authType;
+ return this;
+ }
+
+ public Builder withRemoteMetastore() {
+ this.isMetastoreRemote = true;
+ return this;
+ }
+
+ public Builder withConf(HiveConf hiveConf) {
+ this.hiveConf = hiveConf;
+ return this;
+ }
+
+ public Builder withHA() {
+ this.isHA = true;
+ return this;
+ }
+
+ /**
+ * Start HS2 with HTTP transport mode, default is binary mode
+ * @return this Builder
+ */
+ public Builder withHTTPTransport(){
+ this.isHTTPTransMode = true;
+ return this;
+ }
+
+ public Builder cleanupLocalDirOnStartup(boolean val) {
+ this.cleanupLocalDirOnStartup = val;
+ return this;
+ }
+
+ public MiniHS2 build() throws Exception {
+ if (miniClusterType == MiniClusterType.MR && useMiniKdc) {
+ throw new IOException("Can't create secure miniMr ... yet");
+ }
+ if (isHTTPTransMode) {
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE);
+ } else {
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE);
+ }
+ return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab,
+ isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup);
+ }
+ }
+
+ public MiniMrShim getMr() {
+ return mr;
+ }
+
+ public void setMr(MiniMrShim mr) {
+ this.mr = mr;
+ }
+
+ public MiniDFSShim getDfs() {
+ return dfs;
+ }
+
+ public void setDfs(MiniDFSShim dfs) {
+ this.dfs = dfs;
+ }
+
+ public FileSystem getLocalFS() {
+ return localFS;
+ }
+
+ public MiniClusterType getMiniClusterType() {
+ return miniClusterType;
+ }
+
+ public void setMiniClusterType(MiniClusterType miniClusterType) {
+ this.miniClusterType = miniClusterType;
+ }
+
+ public boolean isUseMiniKdc() {
+ return useMiniKdc;
+ }
+
+ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc,
+ String serverPrincipal, String serverKeytab, boolean isMetastoreRemote,
+ boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception {
+ // Always use localhost for hostname as some tests like SSL CN validation ones
+ // are tied to localhost being present in the certificate name
+ super(
+ hiveConf,
+ "localhost",
+ (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT) : MetaStoreUtils
+ .findFreePort()),
+ (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreUtils
+ .findFreePort()));
+ hiveConf.setLongVar(ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS, 3l);
+ hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS, 10,
+ TimeUnit.SECONDS);
+ this.miniClusterType = miniClusterType;
+ this.useMiniKdc = useMiniKdc;
+ this.serverPrincipal = serverPrincipal;
+ this.isMetastoreRemote = isMetastoreRemote;
+ this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup;
+ baseDir = getBaseDir();
+ localFS = FileSystem.getLocal(hiveConf);
+ FileSystem fs;
+
+ if (miniClusterType != MiniClusterType.LOCALFS_ONLY) {
+ // Initialize dfs
+ dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA);
+ fs = dfs.getFileSystem();
+ String uriString = fs.getUri().toString();
+
+ // Initialize the execution engine based on cluster type
+ switch (miniClusterType) {
+ case TEZ:
+ // Change the engine to tez
+ hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
+ // TODO: This should be making use of confDir to load configs setup for Tez, etc.
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, false);
+ break;
+ case LLAP:
+ if (usePortsFromConf) {
+ hiveConf.setBoolean("minillap.usePortsFromConf", true);
+ }
+ llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null);
+
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, true);
+ break;
+ case MR:
+ mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported cluster type " + mr);
+ }
+ // store the config in system properties
+ mr.setupConfiguration(getHiveConf());
+ baseFsDir = new Path(new Path(fs.getUri()), "/base");
+ } else {
+ // This is FS only mode, just initialize the dfs root directory.
+ fs = FileSystem.getLocal(hiveConf);
+ baseFsDir = new Path("file://" + baseDir.toURI().getPath());
+
+ if (cleanupLocalDirOnStartup) {
+ // Cleanup baseFsDir since it can be shared across tests.
+ LOG.info("Attempting to cleanup baseFsDir: {} while setting up MiniHS2", baseDir);
+ Preconditions.checkState(baseFsDir.depth() >= 3); // Avoid "/tmp", directories closer to "/"
+ fs.delete(baseFsDir, true);
+ }
+ }
+ if (useMiniKdc) {
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal);
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab);
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType);
+ }
+ String metaStoreURL =
+ "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator
+ + "test_metastore;create=true";
+
+ fs.mkdirs(baseFsDir);
+ Path wareHouseDir = new Path(baseFsDir, "warehouse");
+ // Create warehouse with 777, so that user impersonation has no issues.
+ FileSystem.mkdirs(fs, wareHouseDir, FULL_PERM);
+
+ fs.mkdirs(wareHouseDir);
+ setWareHouseDir(wareHouseDir.toString());
+ System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL);
+ hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, metaStoreURL);
+ if (!usePortsFromConf) {
+ // reassign a new port, just in case if one of the MR services grabbed the last one
+ setBinaryPort(MetaStoreUtils.findFreePort());
+ }
+ hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, getHost());
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort());
+ hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort());
+
+ Path scratchDir = new Path(baseFsDir, "scratch");
+ // Create root scratchdir with write all, so that user impersonation has no issues.
+ Utilities.createDirsWithPermission(hiveConf, scratchDir, WRITE_ALL_PERM, true);
+ System.setProperty(HiveConf.ConfVars.SCRATCHDIR.varname, scratchDir.toString());
+ hiveConf.setVar(ConfVars.SCRATCHDIR, scratchDir.toString());
+
+ String localScratchDir = baseDir.getPath() + File.separator + "scratch";
+ System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.varname, localScratchDir);
+ hiveConf.setVar(ConfVars.LOCALSCRATCHDIR, localScratchDir);
+ }
+
+ public MiniHS2(HiveConf hiveConf) throws Exception {
+ this(hiveConf, MiniClusterType.LOCALFS_ONLY);
+ }
+
+ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception {
+ this(hiveConf, clusterType, false);
+ }
+
+ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType,
+ boolean usePortsFromConf) throws Exception {
+ this(hiveConf, clusterType, false, null, null, false, usePortsFromConf,
+ "KERBEROS", false, true);
+ }
+
+ public void start(Map<String, String> confOverlay) throws Exception {
+ if (isMetastoreRemote) {
+ int metaStorePort = MetaStoreUtils.findFreePort();
+ getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
+ MetaStoreUtils.startMetaStore(metaStorePort,
+ ShimLoader.getHadoopThriftAuthBridge(), getHiveConf());
+ }
+
+ hiveServer2 = new HiveServer2();
+ // Set confOverlay parameters
+ for (Map.Entry<String, String> entry : confOverlay.entrySet()) {
+ setConfProperty(entry.getKey(), entry.getValue());
+ }
+ hiveServer2.init(getHiveConf());
+ hiveServer2.start();
+ waitForStartup();
+ setStarted(true);
+ }
+
+ public void stop() {
+ verifyStarted();
+ // Currently there is no way to stop the MetaStore service. It will be stopped when the
+ // test JVM exits. This is how other tests are also using MetaStore server.
+
+ hiveServer2.stop();
+ setStarted(false);
+ try {
+ if (llapCluster != null) {
+ llapCluster.stop();
+ }
+ if (mr != null) {
+ mr.shutdown();
+ mr = null;
+ }
+ if (dfs != null) {
+ dfs.shutdown();
+ dfs = null;
+ }
+ } catch (IOException e) {
+ // Ignore errors cleaning up miniMR
+ }
+ }
+
+ public void cleanup() {
+ FileUtils.deleteQuietly(baseDir);
+ }
+
+
+ public CLIServiceClient getServiceClient() {
+ verifyStarted();
+ return getServiceClientInternal();
+ }
+
+ public HiveConf getServerConf() {
+ if (hiveServer2 != null) {
+ return hiveServer2.getHiveConf();
+ }
+ return null;
+ }
+
+ public CLIServiceClient getServiceClientInternal() {
+ for (Service service : hiveServer2.getServices()) {
+ if (service instanceof ThriftBinaryCLIService) {
+ return new ThriftCLIServiceClient((ThriftBinaryCLIService) service);
+ }
+ if (service instanceof ThriftHttpCLIService) {
+ return new ThriftCLIServiceClient((ThriftHttpCLIService) service);
+ }
+ }
+ throw new IllegalStateException("HiveServer2 not running Thrift service");
+ }
+
+ /**
+ * return connection URL for this server instance
+ * @return
+ * @throws Exception
+ */
+ public String getJdbcURL() throws Exception {
+ return getJdbcURL("default");
+ }
+
+ /**
+ * return connection URL for this server instance
+ * @param dbName - DB name to be included in the URL
+ * @return
+ * @throws Exception
+ */
+ public String getJdbcURL(String dbName) throws Exception {
+ return getJdbcURL(dbName, "");
+ }
+
+ /**
+ * return connection URL for this server instance
+ * @param dbName - DB name to be included in the URL
+ * @param sessionConfExt - Addional string to be appended to sessionConf part of url
+ * @return
+ * @throws Exception
+ */
+ public String getJdbcURL(String dbName, String sessionConfExt) throws Exception {
+ return getJdbcURL(dbName, sessionConfExt, "");
+ }
+
+ /**
+ * return connection URL for this server instance
+ * @param dbName - DB name to be included in the URL
+ * @param sessionConfExt - Addional string to be appended to sessionConf part of url
+ * @param hiveConfExt - Additional string to be appended to HiveConf part of url (excluding the ?)
+ * @return
+ * @throws Exception
+ */
+ public String getJdbcURL(String dbName, String sessionConfExt, String hiveConfExt)
+ throws Exception {
+ sessionConfExt = (sessionConfExt == null ? "" : sessionConfExt);
+ hiveConfExt = (hiveConfExt == null ? "" : hiveConfExt);
+ // Strip the leading ";" if provided
+ // (this is the assumption with which we're going to start configuring sessionConfExt)
+ if (sessionConfExt.startsWith(";")) {
+ sessionConfExt = sessionConfExt.substring(1);
+ }
+ if (isUseMiniKdc()) {
+ sessionConfExt = "principal=" + serverPrincipal + ";" + sessionConfExt;
+ }
+ if (isHttpTransportMode()) {
+ sessionConfExt = "transportMode=http;httpPath=cliservice" + ";" + sessionConfExt;
+ }
+ String baseJdbcURL;
+ if (isDynamicServiceDiscovery()) {
+ sessionConfExt =
+ "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace="
+ + getServerConf().getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE) + ";"
+ + sessionConfExt;
+ baseJdbcURL = getZKBaseJdbcURL();
+ } else {
+ baseJdbcURL = getBaseJdbcURL();
+ }
+
+ baseJdbcURL = baseJdbcURL + dbName;
+ if (!sessionConfExt.isEmpty()) {
+ baseJdbcURL = baseJdbcURL + ";" + sessionConfExt;
+ }
+ if ((hiveConfExt != null) && (!hiveConfExt.trim().isEmpty())) {
+ baseJdbcURL = baseJdbcURL + "?" + hiveConfExt;
+ }
+ return baseJdbcURL;
+ }
+
+ /**
+ * Build base JDBC URL
+ * @return
+ */
+ public String getBaseJdbcURL() {
+ if(isHttpTransportMode()) {
+ return "jdbc:hive2://" + getHost() + ":" + getHttpPort() + "/";
+ }
+ else {
+ return "jdbc:hive2://" + getHost() + ":" + getBinaryPort() + "/";
+ }
+ }
+
+ /**
+ * Build zk base JDBC URL
+ * @return
+ */
+ private String getZKBaseJdbcURL() throws Exception {
+ HiveConf hiveConf = getServerConf();
+ if (hiveConf != null) {
+ String zkEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
+ return "jdbc:hive2://" + zkEnsemble + "/";
+ }
+ throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
+ }
+
+ private boolean isHttpTransportMode() {
+ String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname);
+ return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE));
+ }
+
+ private boolean isDynamicServiceDiscovery() throws Exception {
+ HiveConf hiveConf = getServerConf();
+ if (hiveConf == null) {
+ throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs.");
+ }
+ if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
+ return true;
+ }
+ return false;
+ }
+
+ public static String getJdbcDriverName() {
+ return driverName;
+ }
+
+ public MiniMrShim getMR() {
+ return mr;
+ }
+
+ public MiniDFSShim getDFS() {
+ return dfs;
+ }
+
+ private void waitForStartup() throws Exception {
+ int waitTime = 0;
+ long startupTimeout = 1000L * 1000L;
+ CLIServiceClient hs2Client = getServiceClientInternal();
+ SessionHandle sessionHandle = null;
+ do {
+ Thread.sleep(500L);
+ waitTime += 500L;
+ if (waitTime > startupTimeout) {
+ throw new TimeoutException("Couldn't access new HiveServer2: " + getJdbcURL());
+ }
+ try {
+ Map <String, String> sessionConf = new HashMap<String, String>();
+ /**
+ if (isUseMiniKdc()) {
+ getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal());
+ sessionConf.put("principal", serverPrincipal);
+ }
+ */
+ sessionHandle = hs2Client.openSession("foo", "bar", sessionConf);
+ } catch (Exception e) {
+ // service not started yet
+ continue;
+ }
+ hs2Client.closeSession(sessionHandle);
+ break;
+ } while (true);
+ }
+
+ public Service.STATE getState() {
+ return hiveServer2.getServiceState();
+ }
+
+ static File getBaseDir() {
+ File baseDir = new File(tmpDir + "/local_base");
+ return baseDir;
+ }
+
+ public static void cleanupLocalDir() throws IOException {
+ File baseDir = getBaseDir();
+ try {
+ org.apache.hadoop.hive.common.FileUtils.deleteDirectory(baseDir);
+ } catch (FileNotFoundException e) {
+ // Ignore. Safe if it does not exist.
+ }
+ }
+}