You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2016/05/10 16:48:38 UTC
hive git commit: HIVE-13676 : Tests failing because metastore doesn't
come up (Rajat Khandelwal via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 882a7f000 -> 0ea9d35de
HIVE-13676 : Tests failing because metastore doesn't come up (Rajat Khandelwal via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ea9d35d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ea9d35d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ea9d35d
Branch: refs/heads/master
Commit: 0ea9d35de39df94cb7c7ff4274c7ea4b177ba489
Parents: 882a7f0
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue May 10 01:32:00 2016 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue May 10 09:47:31 2016 -0700
----------------------------------------------------------------------
.../mapreduce/TestHCatMultiOutputFormat.java | 27 +++--------
.../hive/hcatalog/api/TestHCatClient.java | 49 ++++----------------
.../hive/metastore/TestMarkPartitionRemote.java | 26 +----------
.../hadoop/hive/metastore/MetaStoreUtils.java | 26 +++++++++--
.../metastore/TestHiveMetaStoreGetMetaConf.java | 23 ++-------
.../TestHiveMetaStorePartitionSpecs.java | 22 ++-------
.../hadoop/hive/ql/metadata/TestHiveRemote.java | 34 +-------------
7 files changed, 48 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
index 9fa263d..6ff48ee 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
@@ -33,8 +33,8 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -89,7 +89,7 @@ public class TestHCatMultiOutputFormat {
private static HiveConf hiveConf;
private static File workDir;
- private static final String msPort = "20199";
+ private static int msPort;
private static Thread t;
static {
@@ -98,22 +98,6 @@ public class TestHCatMultiOutputFormat {
schemaMap.put(tableNames[2], new HCatSchema(ColumnHolder.hCattest3Cols));
}
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- String warehouseConf = HiveConf.ConfVars.METASTOREWAREHOUSE.varname + "="
- + warehousedir.toString();
- HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", warehouseConf});
- } catch (Throwable t) {
- System.err.println("Exiting. Got exception from metastore: " + t.getMessage());
- t.printStackTrace();
- }
- }
-
- }
-
/**
* Private class which holds all the data for the test cases
*/
@@ -173,10 +157,11 @@ public class TestHCatMultiOutputFormat {
warehousedir = new Path(System.getProperty("test.warehouse.dir"));
- // Run hive metastore server
- t = new Thread(new RunMS());
- t.start();
+ HiveConf metastoreConf = new HiveConf();
+ metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString());
+ // Run hive metastore server
+ msPort = MetaStoreUtils.startMetaStore(metastoreConf);
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
Configuration conf = new Configuration(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
index aa9c7d3..48ee7cf 100644
--- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
+++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
@@ -34,8 +34,8 @@ import com.google.common.collect.Iterables;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
@@ -83,41 +83,14 @@ import javax.annotation.Nullable;
public class TestHCatClient {
private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class);
- private static final String msPort = "20101";
+ private static int msPort;
private static HiveConf hcatConf;
private static boolean isReplicationTargetHCatRunning = false;
- private static final String replicationTargetHCatPort = "20102";
+ private static int replicationTargetHCatPort;
private static HiveConf replicationTargetHCatConf;
private static SecurityManager securityManager;
private static boolean useExternalMS = false;
- public static class RunMS implements Runnable {
-
- private final String msPort;
- private List<String> args = new ArrayList<String>();
-
- public RunMS(String msPort) {
- this.msPort = msPort;
- this.args.add("-v");
- this.args.add("-p");
- this.args.add(this.msPort);
- }
-
- public RunMS arg(String arg) {
- this.args.add(arg);
- return this;
- }
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(args.toArray(new String[args.size()]));
- } catch (Throwable t) {
- LOG.error("Exiting. Got exception from metastore: ", t);
- }
- }
- } // class RunMS;
-
@AfterClass
public static void tearDown() throws Exception {
if (!useExternalMS) {
@@ -142,10 +115,7 @@ public class TestHCatClient {
System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
DbNotificationListener.class.getName()); // turn on db notification listener on metastore
- Thread t = new Thread(new RunMS(msPort));
- t.start();
- Thread.sleep(10000);
-
+ msPort = MetaStoreUtils.startMetaStore();
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
@@ -832,13 +802,10 @@ public class TestHCatClient {
private void startReplicationTargetMetaStoreIfRequired() throws Exception {
if (!isReplicationTargetHCatRunning) {
- Thread t = new Thread(new RunMS(replicationTargetHCatPort)
- .arg("--hiveconf")
- .arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance.
- .arg(hcatConf.get("javax.jdo.option.ConnectionURL")
- .replace("metastore", "target_metastore")));
- t.start();
- Thread.sleep(10000);
+ HiveConf conf = new HiveConf();
+ conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL")
+ .replace("metastore", "target_metastore"));
+ replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf);
replicationTargetHCatConf = new HiveConf(hcatConf);
replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS,
"thrift://localhost:" + replicationTargetHCatPort);
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java
index 7576f39..239bad0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java
@@ -19,37 +19,13 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
public class TestMarkPartitionRemote extends TestMarkPartition {
- private static class RunMS implements Runnable {
-
- private final int port;
-
- public RunMS(int port) {
- this.port = port;
- }
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[] { String.valueOf(port) });
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
-
- }
@Override
protected void setUp() throws Exception {
super.setUp();
- int port = MetaStoreUtils.findFreePort();
- Thread t = new Thread(new RunMS(port));
- t.setDaemon(true);
- t.start();
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore());
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
- Thread.sleep(30000);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index da3da8b..e01fe45 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1176,22 +1176,38 @@ public class MetaStoreUtils {
} catch (IOException e) {
throw new MetaException("Unable to : " + path);
}
+ }
+ public static int startMetaStore() throws Exception {
+ return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), null);
}
- public static void startMetaStore(final int port,
- final HadoopThriftAuthBridge bridge) throws Exception {
- startMetaStore(port, bridge, new HiveConf(HMSHandler.class));
+ public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception {
+ int port = findFreePort();
+ startMetaStore(port, bridge, conf);
+ return port;
+ }
+
+ public static int startMetaStore(HiveConf conf) throws Exception {
+ return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf);
+ }
+
+ public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception {
+ startMetaStore(port, bridge, null);
}
public static void startMetaStore(final int port,
- final HadoopThriftAuthBridge bridge, final HiveConf hiveConf)
+ final HadoopThriftAuthBridge bridge, HiveConf hiveConf)
throws Exception{
+ if (hiveConf == null) {
+ hiveConf = new HiveConf(HMSHandler.class);
+ }
+ final HiveConf finalHiveConf = hiveConf;
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
- HiveMetaStore.startMetaStore(port, bridge, hiveConf);
+ HiveMetaStore.startMetaStore(port, bridge, finalHiveConf);
} catch (Throwable e) {
LOG.error("Metastore Thrift Server threw an exception...",e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
index 3f4561c..da883d2 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java
@@ -40,7 +40,6 @@ public class TestHiveMetaStoreGetMetaConf {
public ExpectedException thrown = ExpectedException.none();
private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStoreGetMetaConf.class);
- private static final String msPort = "20103";
private static HiveConf hiveConf;
private static SecurityManager securityManager;
@@ -65,20 +64,6 @@ public class TestHiveMetaStoreGetMetaConf {
}
}
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf",
- "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName(),
- "--hiveconf", "hive.metastore.try.direct.sql.ddl=false"});
- } catch (Throwable t) {
- LOG.error("Exiting. Got exception from metastore: ", t);
- }
- }
- }
-
@AfterClass
public static void tearDown() throws Exception {
LOG.info("Shutting down metastore.");
@@ -90,7 +75,11 @@ public class TestHiveMetaStoreGetMetaConf {
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
-
+ HiveConf metastoreConf = new HiveConf();
+ metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
+ MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class);
+ metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false);
+ int msPort = MetaStoreUtils.startMetaStore(metastoreConf);
hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
+ msPort);
@@ -101,8 +90,6 @@ public class TestHiveMetaStoreGetMetaConf {
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
-
- new Thread(new RunMS()).start();
}
@Before
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
index 922a4bf..c1646c3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java
@@ -49,7 +49,7 @@ import java.util.Map;
public class TestHiveMetaStorePartitionSpecs {
private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStorePartitionSpecs.class);
- private static final String msPort = "20102";
+ private static int msPort;
private static HiveConf hiveConf;
private static SecurityManager securityManager;
@@ -73,18 +73,6 @@ public class TestHiveMetaStorePartitionSpecs {
}
}
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf",
- "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName()});
- } catch (Throwable t) {
- LOG.error("Exiting. Got exception from metastore: ", t);
- }
- }
- }
@AfterClass
public static void tearDown() throws Exception {
@@ -95,10 +83,10 @@ public class TestHiveMetaStorePartitionSpecs {
@BeforeClass
public static void startMetaStoreServer() throws Exception {
- Thread t = new Thread(new RunMS());
- t.start();
- Thread.sleep(10000);
-
+ HiveConf metastoreConf = new HiveConf();
+ metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
+ MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class);
+ msPort = MetaStoreUtils.startMetaStore(metastoreConf);
securityManager = System.getSecurityManager();
System.setSecurityManager(new NoExitSecurityManager());
hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
index ee95e16..9dfcd54 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.net.ServerSocket;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.util.StringUtils;
/**
@@ -34,43 +34,13 @@ import org.apache.hadoop.util.StringUtils;
public class TestHiveRemote extends TestHive {
/**
- * Starts a remote metastore
- */
- private static class RunMS implements Runnable {
- String port;
-
- public RunMS(String port) {
- this.port = port;
- }
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[] { port });
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
-
- }
-
- /**
* Start a remote metastore and initialize a Hive object pointing at it.
*/
@Override
protected void setUp() throws Exception {
super.setUp();
hiveConf = new HiveConf(this.getClass());
- String port = findFreePort();
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
-
- Thread t = new Thread(new RunMS(port));
- t.start();
-
- // Wait a little bit for the metastore to start.
- Thread.sleep(5000);
-
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore());
try {
hm = Hive.get(hiveConf);