You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/12/11 23:33:15 UTC
svn commit: r1420473 - in /hive/branches/branch-0.10/metastore/src:
java/org/apache/hadoop/hive/metastore/ test/org/apache/hadoop/hive/metastore/
Author: hashutosh
Date: Tue Dec 11 22:33:11 2012
New Revision: 1420473
URL: http://svn.apache.org/viewvc?rev=1420473&view=rev
Log:
HIVE-3724 : Metastore tests use hardcoded ports (Kevin Wilfong, Sushanth Sowmyan via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.10/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
Modified: hive/branches/branch-0.10/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/branch-0.10/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Tue Dec 11 22:33:11 2012
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
+import java.net.ServerSocket;
import java.net.Socket;
import java.util.ArrayList;
import java.util.HashMap;
@@ -40,12 +41,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -801,6 +802,19 @@ public class MetaStoreUtils {
}
/**
+ * Finds a free port on the machine.
+ *
+ * @return
+ * @throws IOException
+ */
+ public static int findFreePort() throws IOException {
+ ServerSocket socket= new ServerSocket(0);
+ int port = socket.getLocalPort();
+ socket.close();
+ return port;
+ }
+
+ /**
* Catches exceptions that can't be handled and bundles them to MetaException
*
* @param e
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java Tue Dec 11 22:33:11 2012
@@ -28,29 +28,16 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
/**
* TestMetaStoreEventListener. Test case for
* {@link org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener}
*/
public class TestMetaStoreEndFunctionListener extends TestCase {
- private static final String msPort = "20002";
private HiveConf hiveConf;
private HiveMetaStoreClient msc;
private Driver driver;
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[]{msPort});
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
- }
-
@Override
protected void setUp() throws Exception {
@@ -61,11 +48,11 @@ public class TestMetaStoreEndFunctionLis
DummyPreListener.class.getName());
System.setProperty("hive.metastore.end.function.listeners",
DummyEndFunctionListener.class.getName());
- Thread t = new Thread(new RunMS());
- t.start();
- Thread.sleep(40000);
+ int port = MetaStoreUtils.findFreePort();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+
hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java Tue Dec 11 22:33:11 2012
@@ -27,7 +27,6 @@ import junit.framework.TestCase;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
@@ -54,6 +53,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
/**
* TestMetaStoreEventListener. Test case for
@@ -61,7 +61,6 @@ import org.apache.hadoop.hive.ql.session
* {@link org.apache.hadoop.hive.metastore.MetaStorePreEventListener}
*/
public class TestMetaStoreEventListener extends TestCase {
- private static final String msPort = "20001";
private HiveConf hiveConf;
private HiveMetaStoreClient msc;
private Driver driver;
@@ -70,19 +69,6 @@ public class TestMetaStoreEventListener
private static final String tblName = "tmptbl";
private static final String renamed = "tmptbl2";
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[]{msPort});
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
- }
-
@Override
protected void setUp() throws Exception {
@@ -93,12 +79,11 @@ public class TestMetaStoreEventListener
System.setProperty("hive.metastore.pre.event.listeners",
DummyPreListener.class.getName());
- Thread t = new Thread(new RunMS());
- t.start();
- Thread.sleep(40000);
+ int port = MetaStoreUtils.findFreePort();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java Tue Dec 11 22:33:11 2012
@@ -20,10 +20,10 @@ package org.apache.hadoop.hive.metastore
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.ShimLoader;
public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
- protected static final String METASTORE_PORT = "29083";
private static boolean isServerStarted = false;
public TestRemoteHiveMetaStore() {
@@ -31,20 +31,6 @@ public class TestRemoteHiveMetaStore ext
isThriftClient = true;
}
- private static class RunMS implements Runnable {
-
- @Override
- public void run() {
- try {
- HiveMetaStore.main(new String[] { METASTORE_PORT });
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
-
- }
-
@Override
protected void setUp() throws Exception {
super.setUp();
@@ -54,20 +40,17 @@ public class TestRemoteHiveMetaStore ext
return;
}
- System.out.println("Starting MetaStore Server on port " + METASTORE_PORT);
- Thread t = new Thread(new RunMS());
- t.start();
+ int port = MetaStoreUtils.findFreePort();
+ System.out.println("Starting MetaStore Server on port " + port);
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
isServerStarted = true;
- // Wait a little bit for the metastore to start. Should probably have
- // a better way of detecting if the metastore has started?
- Thread.sleep(5000);
// This is default case with setugi off for both client and server
- createClient(false);
+ createClient(false, port);
}
- protected void createClient(boolean setugi) throws Exception {
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
+ protected void createClient(boolean setugi, int port) throws Exception {
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI,setugi);
client = new HiveMetaStoreClient(hiveConf);
}
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java Tue Dec 11 22:33:11 2012
@@ -18,14 +18,11 @@
package org.apache.hadoop.hive.metastore;
-import java.io.IOException;
-import java.net.ServerSocket;
-
import junit.framework.TestCase;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.StringUtils;
/**
@@ -40,50 +37,23 @@ public class TestRemoteHiveMetaStoreIpAd
private static HiveConf hiveConf;
private static HiveMetaStoreClient msc;
- private String port;
-
- private static class RunMS implements Runnable {
- String port;
-
- public RunMS(String port) {
- this.port = port;
- }
-
- @Override
- public void run() {
- try {
- System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname,
- IpAddressListener.class.getName());
- HiveMetaStore.main(new String[] { port });
- } catch (Throwable e) {
- e.printStackTrace(System.err);
- assert false;
- }
- }
-
- }
-
@Override
protected void setUp() throws Exception {
super.setUp();
hiveConf = new HiveConf(this.getClass());
- port = findFreePort();
if (isServerStarted) {
assertNotNull("Unable to connect to the MetaStore server", msc);
return;
}
+ int port = MetaStoreUtils.findFreePort();
System.out.println("Starting MetaStore Server on port " + port);
- Thread t = new Thread(new RunMS(port));
- t.start();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
isServerStarted = true;
- // Wait a little bit for the metastore to start. Should probably have
- // a better way of detecting if the metastore has started?
- Thread.sleep(5000);
// This is default case with setugi off for both client and server
- createClient();
+ createClient(port);
}
public void testIpAddress() throws Exception {
@@ -100,15 +70,8 @@ public class TestRemoteHiveMetaStoreIpAd
}
}
- protected void createClient() throws Exception {
+ protected void createClient(int port) throws Exception {
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
msc = new HiveMetaStoreClient(hiveConf);
}
-
- private String findFreePort() throws IOException {
- ServerSocket socket= new ServerSocket(0);
- int port = socket.getLocalPort();
- socket.close();
- return String.valueOf(port);
- }
}
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java Tue Dec 11 22:33:11 2012
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.metastore
public class TestSetUGIOnOnlyClient extends TestRemoteHiveMetaStore{
@Override
- protected void createClient(boolean setugi) throws Exception {
+ protected void createClient(boolean setugi, int port) throws Exception {
// turn it on for client.
- super.createClient(true);
+ super.createClient(true, port);
}
}
Modified: hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java?rev=1420473&r1=1420472&r2=1420473&view=diff
==============================================================================
--- hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java (original)
+++ hive/branches/branch-0.10/metastore/src/test/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java Tue Dec 11 22:33:11 2012
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.metastore
public class TestSetUGIOnOnlyServer extends TestSetUGIOnBothClientServer {
@Override
- protected void createClient(boolean setugi) throws Exception {
+ protected void createClient(boolean setugi, int port) throws Exception {
// It is turned on for both client and server because of super class. Turn it off for client.
- super.createClient(false);
+ super.createClient(false, port);
}
}