incubator-sentry git commit: SENTRY-609: [Unit Test] Many tests failing with error "The root scratch dir: /tmp/hive on HDFS should be writable. Current permissions are: rwxr-xr-x" (Reviewed by Colin Ma)

[sd...@apache.org]

Repository: incubator-sentry
Updated Branches:
  refs/heads/master adf02be4c -> 3f9847128


SENTRY-609: [Unit Test] Many tests failing with error "The root scratch dir: /tmp/hive on HDFS should be writable. Current permissions are: rwxr-xr-x" (Reviewed by Colin Ma)


Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/3f984712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/3f984712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/3f984712

Branch: refs/heads/master
Commit: 3f98471282f002f641b3e0854a83cecb1273bd2b
Parents: adf02be
Author: Sun Dapeng <sd...@apache.org>
Authored: Thu Feb 12 09:48:58 2015 +0800
Committer: Sun Dapeng <sd...@apache.org>
Committed: Thu Feb 12 09:48:58 2015 +0800

----------------------------------------------------------------------
 pom.xml                                           |  3 +++
 .../TestSentryHiveAuthorizationTaskFactory.java   | 17 +++++++++++++++++
 .../org/apache/sentry/binding/hive/TestURI.java   | 18 ++++++++++++++++++
 .../e2e/hive/hiveserver/HiveServerFactory.java    | 15 +++++++++++++++
 .../e2e/hive/hiveserver/InternalHiveServer.java   |  4 +++-
 5 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3f984712/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index abdd093..8905862 100644
--- a/pom.xml
+++ b/pom.xml
@@ -641,6 +641,9 @@ limitations under the License.
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>
             <argLine>-Xms512m -Xmx2g</argLine>
+            <systemPropertyVariables>
+              <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+            </systemPropertyVariables>
           </configuration>
         </plugin>
         <plugin>

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3f984712/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
index eebf419..a50a40d 100644
--- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
+++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
@@ -19,6 +19,7 @@ package org.apache.sentry.binding.hive;
 
 import junit.framework.Assert;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hive.SentryHiveConstants;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -46,10 +47,14 @@ import org.apache.hadoop.hive.ql.plan.ShowGrantDesc;
 import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator;
 import org.apache.hadoop.hive.ql.security.authorization.Privilege;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import com.google.common.io.Files;
+
+import java.io.File;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
@@ -73,13 +78,18 @@ public class TestSentryHiveAuthorizationTaskFactory {
   private Hive db;
   private Table table;
   private Partition partition;
+  private File baseDir;
 
   @Before
   public void setup() throws Exception {
     conf = new HiveConf();
+    baseDir = Files.createTempDir();
+    baseDir.setWritable(true, false);
+    conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
     SessionState.start(conf);
     conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
         SentryHiveAuthorizationTaskFactoryImpl.class.getName());
+
     db = Mockito.mock(Hive.class);
     table = new Table(DB, TABLE);
     partition = new Partition(table);
@@ -450,4 +460,11 @@ public class TestSentryHiveAuthorizationTaskFactory {
     Assert.assertEquals(list.toString(), size, list.size());
     return list;
   }
+
+  @After
+  public void clear() {
+    if(baseDir != null) {
+      FileUtils.deleteQuietly(baseDir);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3f984712/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestURI.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestURI.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestURI.java
index e99d37f..cdd4e0b 100644
--- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestURI.java
+++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestURI.java
@@ -16,20 +16,31 @@
  */
 package org.apache.sentry.binding.hive;
 
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import com.google.common.io.Files;
+
 public class TestURI {
 
   private static HiveConf conf;
+  private static File baseDir;
+
   @BeforeClass
   public static void setupTestURI() {
     conf = new HiveConf();
+    baseDir = Files.createTempDir();
+    baseDir.setWritable(true, false);
+    conf.setVar(HiveConf.ConfVars.SCRATCHDIR, baseDir.getAbsolutePath());
     SessionState.start(conf);
   }
 
@@ -60,4 +71,11 @@ public class TestURI {
     Assert.assertEquals("hdfs:///some/path",
         HiveAuthzBindingHook.parseURI("hdfs:///some/path").getName());
   }
+
+  @AfterClass
+  public static void clear() {
+    if(baseDir != null) {
+      FileUtils.deleteQuietly(baseDir);
+    }
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3f984712/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
index 934ceb8..14f943b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Files;
 import com.google.common.io.Resources;
 
 public class HiveServerFactory {
@@ -111,6 +112,20 @@ public class HiveServerFactory {
         fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777));
       }
     }
+    Boolean policyOnHDFS = new Boolean(System.getProperty("sentry.e2etest.policyonhdfs", "false"));
+    if (policyOnHDFS) {
+      // Initialize "hive.exec.scratchdir", according the description of
+      // "hive.exec.scratchdir", the permission should be (733).
+      // <description>HDFS root scratch dir for Hive jobs which gets created with write
+      // all (733) permission. For each connecting user, an HDFS scratch dir:
+      // ${hive.exec.scratchdir}/&lt;username&gt; is created,
+      // with ${hive.scratch.dir.permission}.</description>
+      fileSystem.mkdirs(new Path("/tmp/hive/"));
+      fileSystem.setPermission(new Path("/tmp/hive/"), new FsPermission((short) 0733));
+    } else {
+      LOGGER.info("Setting an readable path to hive.exec.scratchdir");
+      properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath());
+    }
     if(!properties.containsKey(METASTORE_CONNECTION_URL)) {
       properties.put(METASTORE_CONNECTION_URL,
           String.format("jdbc:derby:;databaseName=%s;create=true",

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3f984712/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/InternalHiveServer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/InternalHiveServer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/InternalHiveServer.java
index a19cbd3..0e53d3d 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/InternalHiveServer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/InternalHiveServer.java
@@ -42,6 +42,8 @@ public class InternalHiveServer extends AbstractHiveServer {
 
   @Override
   public synchronized void shutdown() throws Exception {
-    hiveServer2.stop();
+    if (hiveServer2 != null) {
+      hiveServer2.stop();
+    }
   }
 }