You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by as...@apache.org on 2014/11/19 08:26:47 UTC
incubator-sentry git commit: SENTRY-525: Fix TestHDFSIntegration test
failure
Repository: incubator-sentry
Updated Branches:
refs/heads/master 32e671559 -> d540ad385
SENTRY-525: Fix TestHDFSIntegration test failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/d540ad38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/d540ad38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/d540ad38
Branch: refs/heads/master
Commit: d540ad385d77d9847c3e9ddccc81c48e589ee833
Parents: 32e6715
Author: Arun Suresh <as...@cloudera.com>
Authored: Tue Nov 18 23:26:08 2014 -0800
Committer: Arun Suresh <as...@cloudera.com>
Committed: Tue Nov 18 23:26:08 2014 -0800
----------------------------------------------------------------------
.../tests/e2e/hdfs/TestHDFSIntegration.java | 82 ++++++++++++++------
1 file changed, 59 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/d540ad38/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
index a488c94..30c72eb 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import junit.framework.Assert;
@@ -89,12 +90,17 @@ import org.fest.reflect.core.Reflection;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.io.Resources;
public class TestHDFSIntegration {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(TestHDFSIntegration.class);
public static class WordCountMapper extends MapReduceBase implements
Mapper<LongWritable, Text, String, Long> {
@@ -194,6 +200,10 @@ public class TestHDFSIntegration {
}
private void startHiveAndMetastore() throws IOException, InterruptedException {
+ startHiveAndMetastore(NUM_RETRIES);
+ }
+
+ private void startHiveAndMetastore(final int retries) throws IOException, InterruptedException {
hiveUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
@@ -228,7 +238,7 @@ public class TestHDFSIntegration {
hiveConf.set("datanucleus.fixedDatastore", "false");
hiveConf.set("datanucleus.autoStartMechanism", "SchemaTable");
hmsPort = findPort();
- System.out.println("\n\n HMS port : " + hmsPort + "\n\n");
+ LOGGER.info("\n\n HMS port : " + hmsPort + "\n\n");
hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort);
hiveConf.set("hive.metastore.pre.event.listeners", "org.apache.sentry.binding.metastore.MetastoreAuthzBinding");
hiveConf.set("hive.metastore.event.listeners", "org.apache.sentry.binding.metastore.SentryMetastorePostEventListener");
@@ -245,7 +255,7 @@ public class TestHDFSIntegration {
out.close();
hiveConf.set("hive.sentry.conf.url", accessSite.getPath());
- System.out.println("Sentry client file : " + accessSite.getPath());
+ LOGGER.info("Sentry client file : " + accessSite.getPath());
File hiveSite = new File(confDir, "hive-site.xml");
hiveConf.set("hive.server2.enable.doAs", "false");
@@ -268,30 +278,55 @@ public class TestHDFSIntegration {
metastore.start();
while(true){}
} catch (Exception e) {
- System.out.println("Could not start Hive Server");
- }
- }
- }.start();
-
- hiveServer2 = new InternalHiveServer(hiveConf);
- new Thread() {
- @Override
- public void run() {
- try {
- hiveServer2.start();
- while(true){}
- } catch (Exception e) {
- System.out.println("Could not start Hive Server");
+ LOGGER.info("Could not start Hive Server");
}
}
}.start();
- Thread.sleep(10000);
+ startHiveServer2(retries, hiveConf);
return null;
}
});
}
+ private void startHiveServer2(final int retries, HiveConf hiveConf)
+ throws IOException, InterruptedException, SQLException {
+ Connection conn = null;
+ Thread th = null;
+ final AtomicBoolean keepRunning = new AtomicBoolean(true);
+ try {
+ hiveServer2 = new InternalHiveServer(hiveConf);
+ th = new Thread() {
+ @Override
+ public void run() {
+ try {
+ hiveServer2.start();
+ while(keepRunning.get()){}
+ } catch (Exception e) {
+ LOGGER.info("Could not start Hive Server");
+ }
+ }
+ };
+ th.start();
+ Thread.sleep(RETRY_WAIT * 5);
+ conn = hiveServer2.createConnection("hive", "hive");
+ } catch (Exception ex) {
+ if (retries > 0) {
+ try {
+ keepRunning.set(false);
+ hiveServer2.shutdown();
+ } catch (Exception e) {
+ // Ignore
+ }
+ LOGGER.info("Re-starting Hive Server2 !!");
+ startHiveServer2(retries - 1, hiveConf);
+ }
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+
private void startDFSandYARN() throws IOException,
InterruptedException {
adminUgi.doAs(new PrivilegedExceptionAction<Void>() {
@@ -323,8 +358,8 @@ public class TestHDFSIntegration {
Path warehousePath = new Path(hivePath, "warehouse");
miniDFS.getFileSystem().mkdirs(warehousePath);
boolean directory = miniDFS.getFileSystem().isDirectory(warehousePath);
- System.out.println("\n\n Is dir :" + directory + "\n\n");
- System.out.println("\n\n DefaultFS :" + miniDFS.getFileSystem().getUri() + "\n\n");
+ LOGGER.info("\n\n Is dir :" + directory + "\n\n");
+ LOGGER.info("\n\n DefaultFS :" + miniDFS.getFileSystem().getUri() + "\n\n");
fsURI = miniDFS.getFileSystem().getUri().toString();
conf.set("fs.defaultFS", fsURI);
@@ -335,12 +370,12 @@ public class TestHDFSIntegration {
miniDFS.getFileSystem().setPermission(tmpPath, FsPermission.valueOf("drwxrwxrwx"));
miniDFS.getFileSystem().setOwner(hivePath, "hive", "hive");
miniDFS.getFileSystem().setOwner(warehousePath, "hive", "hive");
- System.out.println("\n\n Owner :"
+ LOGGER.info("\n\n Owner :"
+ miniDFS.getFileSystem().getFileStatus(warehousePath).getOwner()
+ ", "
+ miniDFS.getFileSystem().getFileStatus(warehousePath).getGroup()
+ "\n\n");
- System.out.println("\n\n Owner tmp :"
+ LOGGER.info("\n\n Owner tmp :"
+ miniDFS.getFileSystem().getFileStatus(tmpPath).getOwner() + ", "
+ miniDFS.getFileSystem().getFileStatus(tmpPath).getGroup() + ", "
+ miniDFS.getFileSystem().getFileStatus(tmpPath).getPermission() + ", "
@@ -351,7 +386,7 @@ public class TestHDFSIntegration {
for (int i = dfsSafeCheckRetry; i > 0; i--) {
if (!miniDFS.getFileSystem().isInSafeMode()) {
hasStarted = true;
- System.out.println("HDFS safemode check num times : " + (31 - i));
+ LOGGER.info("HDFS safemode check num times : " + (31 - i));
break;
}
}
@@ -410,7 +445,7 @@ public class TestHDFSIntegration {
String.valueOf(sentryService.getAddress().getPort()));
waitOnSentryService();
sentryPort = sentryService.getAddress().getPort();
- System.out.println("\n\n Sentry port : " + sentryPort + "\n\n");
+ LOGGER.info("\n\n Sentry port : " + sentryPort + "\n\n");
return null;
}
});
@@ -784,4 +819,5 @@ public class TestHDFSIntegration {
}
}
+
}