You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/20 00:25:36 UTC
[44/58] [abbrv] hive git commit: Revert "HIVE-13496. Create initial
test data once across multiple test runs - TestCliDriver. (Siddharth Seth,
reviewed by Ashutosh Chauhan)"
Revert "HIVE-13496. Create initial test data once across multiple test runs - TestCliDriver. (Siddharth Seth, reviewed by Ashutosh Chauhan)"
This reverts commit 976e628fc01911936caa19e61ea3342f3a19455a.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3e43c6d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3e43c6d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3e43c6d
Branch: refs/heads/llap
Commit: e3e43c6dd89b9643d3f7a8f21c4789af24bba6eb
Parents: 940fc7d
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Apr 14 11:32:19 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Apr 14 11:32:19 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/QTestUtil.java | 247 ++-----------------
pom.xml | 2 -
ql/src/test/templates/TestCliDriver.vm | 38 +--
3 files changed, 22 insertions(+), 265 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 79646cd..2f109ab 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -51,6 +51,7 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -62,7 +63,6 @@ import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -84,7 +84,9 @@ import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapItUtils;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
@@ -97,7 +99,6 @@ import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -138,8 +139,6 @@ public class QTestUtil {
// security property names
private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
private static final String CRLF = System.getProperty("line.separator");
- private static final String TEST_BUILD_DIR = System.getProperty("test.build.dir");
- private static final String CACHED_DATA_DIR_NAME = "cachedData";
private static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
@@ -184,16 +183,6 @@ public class QTestUtil {
private final String cleanupScript;
private boolean useHBaseMetastore = false;
- // Parameters which help tracking cached data generation.
- private final String driverName;
- private Path cachedDataPath;
- private String metaStorePathString;
- private Path metaStorePath;
- private FileSystem localFs;
- private boolean attemptingCacheUsage;
-
- private boolean dbEtcSetup = false;
-
public interface SuiteAddTestFunctor {
public void addTestToSuite(TestSuite suite, Object setup, String tName);
}
@@ -389,34 +378,11 @@ public class QTestUtil {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer, String initScript, String cleanupScript,
- boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
- // For now, to avoid changing multiple test templates, a null driver name avoids
- // data generation optimizations.
- this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
- useHBaseMetastore, withLlapIo, null);
- }
-
- public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript,
- boolean useHBaseMetastore, boolean withLlapIo, String driverName)
+ boolean useHBaseMetastore, boolean withLlapIo)
throws Exception {
- this.attemptingCacheUsage = (StringUtils.isEmpty(TEST_BUILD_DIR) ||
- StringUtils.isEmpty(driverName) || useHBaseMetastore) ? false : true;
- this.driverName = driverName;
this.outDir = outDir;
this.logDir = logDir;
- LOG.info("Creating QTestUtil with settings: "
- + "driverName=" + driverName
- + ", attemptingCacheUsage=" + attemptingCacheUsage
- + ", test.build.dir=" + System.getProperty("test.build.dir")
- + ", useHbaseMetaStore=" + useHBaseMetastore
- + ", withLlapIo=" + withLlapIo
- + ", confDir=" + confDir
- + ", outDir=" + outDir
- + ", logDir=" + logDir
- + ", initScript=" + initScript
- + ", cleanupScript=" + cleanupScript);
this.useHBaseMetastore = useHBaseMetastore;
if (confDir != null && !confDir.isEmpty()) {
@@ -505,7 +471,6 @@ public class QTestUtil {
if (scriptsDir == null) {
scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
}
- LOG.info("Using DataDir=" + dataDir + ", ScriptsDir=" + scriptsDir);
this.initScript = scriptsDir + File.separator + initScript;
this.cleanupScript = scriptsDir + File.separator + cleanupScript;
@@ -867,17 +832,6 @@ public class QTestUtil {
return;
}
- if (!attemptingCacheUsage) {
- cleanupNonCacheUsage();
- } else {
- cleanupCacheUsage();
- }
-
- FunctionRegistry.unregisterTemporaryUDF("test_udaf");
- FunctionRegistry.unregisterTemporaryUDF("test_error");
- }
-
- private void cleanupNonCacheUsage() throws Exception {
clearTablesCreatedDuringTests();
clearKeysCreatedInTests();
@@ -895,42 +849,21 @@ public class QTestUtil {
LOG.info("No cleanup script detected. Skipping.");
}
- cleanupWarehouseDir();
- }
-
- private void cleanupCacheUsage() throws IOException {
- // Remove the Warehouse and metastore directories completely.
- // Also close the current db, since files are going to come in to replace it soon.
- Preconditions.checkState(attemptingCacheUsage);
- Preconditions.checkNotNull(metaStorePath);
- Preconditions.checkNotNull(localFs);
- Hive.closeCurrent();
- cleanupMetastoreDir();
- cleanupWarehouseDir();
- }
-
- private void cleanupWarehouseDir() throws IOException {
// delete any contents in the warehouse dir
Path p = new Path(testWarehouse);
FileSystem fs = p.getFileSystem(conf);
try {
- FileStatus[] ls = fs.listStatus(p);
- for (int i = 0; (ls != null) && (i < ls.length); i++) {
+ FileStatus [] ls = fs.listStatus(p);
+ for (int i=0; (ls != null) && (i<ls.length); i++) {
fs.delete(ls[i].getPath(), true);
}
} catch (FileNotFoundException e) {
// Best effort
}
- }
- private void cleanupMetastoreDir() throws IOException {
- try {
- LOG.info("Cleaning up metastore Dir: {}", metaStorePath);
- localFs.delete(metaStorePath, true);
- } catch (FileNotFoundException e) {
- // Best effort
- }
+ FunctionRegistry.unregisterTemporaryUDF("test_udaf");
+ FunctionRegistry.unregisterTemporaryUDF("test_error");
}
protected void runCreateTableCmd(String createTableCmd) throws Exception {
@@ -960,10 +893,6 @@ public class QTestUtil {
}
public void createSources(String tname) throws Exception {
- createSources(tname, false);
- }
-
- public void createSources(String tname, boolean forceCreate) throws Exception {
boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
if(!isSessionStateStarted) {
startSessionState(canReuseSession);
@@ -972,173 +901,34 @@ public class QTestUtil {
if(cliDriver == null) {
cliDriver = new CliDriver();
}
-
+ cliDriver.processLine("set test.data.dir=" + testFiles + ";");
File scriptFile = new File(this.initScript);
if (!scriptFile.isFile()) {
LOG.info("No init script detected. Skipping");
- if (attemptingCacheUsage) {
- setupDbsEtc(true, true);
- }
return;
}
+ conf.setBoolean("hive.test.init.phase", true);
- if (!attemptingCacheUsage || forceCreate) {
- LOG.info("Creating sources without data caching. attemptingCacheUsage={}, forceCreate={}",
- attemptingCacheUsage, forceCreate);
- cliDriver.processLine("set test.data.dir=" + testFiles + ";");
- conf.setBoolean("hive.test.init.phase", true);
- createSourcesNonCached(scriptFile);
- } else {
- LOG.info("Creating sources with data caching");
- createSourcesCached(scriptFile);
- }
-
- conf.setBoolean("hive.test.init.phase", false);
- }
-
- private void createSourcesNonCached(File scriptFile) throws IOException {
String initCommands = readEntireFileIntoString(scriptFile);
LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
cliDriver.processLine(initCommands);
- }
-
- private void createSourcesCached(File scriptFile) throws IOException, HiveException {
- // First check if the cache already exists. If it does just copy it over.
- Path cachedWarehousePath = new Path(cachedDataPath, "warehouse");
- Path cachedMetaStorePtah = new Path(cachedDataPath, "metastore");
- if (localFs.exists(cachedDataPath)) {
- if (localFs.exists(cachedWarehousePath) && localFs.exists(cachedMetaStorePtah)) {
- LOG.info("Cached data found in {}. Attempting to use it", cachedDataPath);
- // Data is alredy cached
- // Copy the files over to where they should be
- Path warehousePath = new Path(testWarehouse);
- FileSystem warehouseFs = warehousePath.getFileSystem(conf);
- try {
- warehouseFs.delete(warehousePath, false);
- } catch (FileNotFoundException e) {
- // Does not matter if it does not exist.
- }
- warehouseFs.copyFromLocalFile(false, cachedWarehousePath, warehousePath);
-
- try {
- localFs.delete(metaStorePath, false);
- } catch (IOException e) {
- // Does not matter if it does not exist.
- }
- localFs.copyFromLocalFile(false, cachedMetaStorePtah, metaStorePath);
- setupDbsEtc(true, false);
- cliDriver.processLine("set test.data.dir=" + testFiles + ";");
- conf.setBoolean("hive.test.init.phase", true);
-
- return;
- } else {
- // Something is missing. Cleanup. Re-generate and cache
- LOG.info("Partial or no cached data found at {}. Cache will be created", cachedDataPath);
- localFs.delete(cachedDataPath, true);
- }
- } else {
- LOG.info("No cached data found at {}. Cache will be created", cachedDataPath);
- // No caching. Re-generate the data and cache it.
- }
-
- // Generate and cache the data
- setupDbsEtc(true, true);
- cliDriver.processLine("set test.data.dir=" + testFiles + ";");
- conf.setBoolean("hive.test.init.phase", true);
- createSourcesNonCached(scriptFile);
-
- // Close the DB so that contents can be copied out safely.
- Hive.closeCurrent();
-
- // Cache the sources
- localFs.mkdirs(cachedDataPath);
-
- Path warehousePath = new Path(testWarehouse);
- FileSystem warehouseFs = warehousePath.getFileSystem(conf);
-
- warehouseFs.copyToLocalFile(false, warehousePath, cachedWarehousePath, true);
- localFs.copyToLocalFile(false, metaStorePath, cachedMetaStorePtah, true);
-
- // Re-open the DB etc.
- setupDbsEtc(true, false);
- }
-
- private static final Pattern metaStoreUriPattern =
- Pattern.compile("derby.*?databaseName=(.*?)(;|$)");
-
- private String getDerbyDbPath(String jdbcConnectString) {
- if (StringUtils.isEmpty(jdbcConnectString)) {
- return null;
- }
- Matcher matcher = metaStoreUriPattern.matcher(jdbcConnectString);
- if (matcher.find()) {
- return matcher.group(1);
- } else {
- return null;
- }
+ conf.setBoolean("hive.test.init.phase", false);
}
public void init() throws Exception {
- LOG.info("init");
- testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
- LOG.info("TestWarehouseDir set to: [{}]", testWarehouse);
- if (attemptingCacheUsage) {
- // The derby path comes from METASTORECONNECTURLKEY. Default ends up being target/junit_metastore_db
- String metaStoreConnectUrl = conf.getVar(ConfVars.METASTORECONNECTURLKEY);
- LOG.info("MetastoreConnectUrl: " + metaStoreConnectUrl);
- metaStorePathString = getDerbyDbPath(metaStoreConnectUrl);
-
- if (metaStorePathString == null) {
- LOG.warn(
- "Disabling attempted cache usage since metastore path cannot be determined from {}",
- metaStoreConnectUrl);
- attemptingCacheUsage = false;
- } else {
- LOG.info("Metastore url path: " + metaStorePathString);
- metaStorePath = new Path(metaStorePathString);
- if (metaStorePath.isAbsolute() && metaStorePathString.split(File.separator).length >= 3) {
- // Turn this on only if the path is absolute, and is at least 3 deep - since we'll be deleting files later.
- localFs = FileSystem.getLocal(conf).getRaw();
- assert(TEST_BUILD_DIR != null);
- cachedDataPath = new Path(TEST_BUILD_DIR, CACHED_DATA_DIR_NAME);
- cachedDataPath = new Path(cachedDataPath, driverName);
- LOG.info("Using cachedDataPath: " + cachedDataPath);
- } else {
- LOG.warn(
- "Disableing attempted cache usage since metastore path may not be absolute, or depth is < 3. MetaStorePath={}",
- metaStorePathString);
- metaStorePath = null;
- attemptingCacheUsage = false;
- }
- }
- }
+ testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
SessionState.start(conf);
conf.set("hive.execution.engine", execEngine);
-
- if (!attemptingCacheUsage) {
- setupDbsEtc(true, true);
- }
- }
-
- private void setupDbsEtc(boolean force, boolean isNewDb) throws HiveException {
- if (!dbEtcSetup || force) {
- if (isNewDb) {
- db = Hive.get(conf);
- } else {
- db = Hive.getWithFastCheck(conf, false);
- }
- LOG.info("Obtained db");
- drv = new Driver(conf);
- drv.init();
- pd = new ParseDriver();
- sem = new SemanticAnalyzer(conf);
- dbEtcSetup = true;
- }
+ db = Hive.get(conf);
+ drv = new Driver(conf);
+ drv.init();
+ pd = new ParseDriver();
+ sem = new SemanticAnalyzer(conf);
}
public void init(String tname) throws Exception {
@@ -1154,9 +944,8 @@ public class QTestUtil {
public String cliInit(String tname, boolean recreate) throws Exception {
if (recreate) {
cleanUp(tname);
- createSources(tname, true);
+ createSources(tname);
}
- setupDbsEtc(false, true);
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 08ef998..77cfaeb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,7 +75,6 @@
<test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath>
<test.log4j.scheme>file://</test.log4j.scheme>
<test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
- <test.build.dir>${project.build.directory}</test.build.dir>
<test.tmp.dir.uri>file://${test.tmp.dir}</test.tmp.dir.uri>
<test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir>
<test.warehouse.scheme>pfile://</test.warehouse.scheme>
@@ -1027,7 +1026,6 @@
<test.data.dir>${basedir}/${hive.path.to.root}/data/files</test.data.dir>
<test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
<test.tmp.dir.uri>${test.tmp.dir.uri}</test.tmp.dir.uri>
- <test.build.dir>${test.build.dir}</test.build.dir>
<test.dfs.mkdir>${test.dfs.mkdir}</test.dfs.mkdir>
<test.output.overwrite>${test.output.overwrite}</test.output.overwrite>
<test.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</test.warehouse.dir>
http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index 1961c75..72cfab9 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -17,34 +17,23 @@
*/
package org.apache.hadoop.hive.cli;
-import com.google.common.base.Stopwatch;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.util.concurrent.TimeUnit;
-
public class $className {
- private static final Logger LOG = LoggerFactory.getLogger(${className}.class);
-
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
static {
- Stopwatch stopwatch = new Stopwatch().start();
- String message = "Starting TestCliDriver run at " + System.currentTimeMillis();
- LOG.info(message);
- System.err.println(message);
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
@@ -55,29 +44,15 @@ public class $className {
if (!hiveConfDir.isEmpty()) {
hiveConfDir = HIVE_ROOT + hiveConfDir;
}
- // TODO Is ZK startup required for TestCliDriver
- // TODO Is LlapIo enabled required for TestCliDriver
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, "$className");
- message = "QTestUtil instance created. ElapsedTimeSinceStart=" + stopwatch.elapsed(
- TimeUnit.MILLISECONDS);
- LOG.info(message);
- System.err.println(message);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
// do a one time initialization
qt.cleanUp();
- message = "Initialization cleanup done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS);
- LOG.info(message);
- System.err.println(message);
-
qt.createSources();
- message = "Initialization createSources done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS);
- LOG.info(message);
- System.err.println(message);
} catch (Exception e) {
- System.err.println("Exception: " + e.getMessage() + ". ElapsedTimeSinceStart="
- + stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
System.err.flush();
fail("Unexpected exception in static initialization: "+e.getMessage());
@@ -87,7 +62,6 @@ public class $className {
@Before
public void setUp() {
try {
- // TODO This restarts ZK for each test. Is that requried ?
qt.clearTestSideEffects();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
@@ -139,9 +113,7 @@ public class $className {
private void runTest(String tname, String fname, String fpath) throws Exception {
long startTime = System.currentTimeMillis();
try {
- String message = "Begin query: " + fname + ", startTime=" + startTime;
- System.err.println(message);
- LOG.info(message);
+ System.err.println("Begin query: " + fname);
qt.addFile(fpath);
@@ -164,9 +136,7 @@ public class $className {
}
long elapsedTime = System.currentTimeMillis() - startTime;
- String message = "Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s";
- System.err.println(message);
- LOG.info(message);
+ System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s");
assertTrue("Test passed", true);
}
}