You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/05 04:32:21 UTC
svn commit: r1622597 [1/2] - in /hive/branches/tez: ./
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/
hcatalog/src/test/e2e/templeton/drivers/
itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/ ite...
Author: gunther
Date: Fri Sep 5 02:32:20 2014
New Revision: 1622597
URL: http://svn.apache.org/r1622597
Log:
Merge latest trunk into branch. (Gunther Hagleitner)
Added:
hive/branches/tez/ql/src/test/queries/clientpositive/uber_reduce.q
- copied unchanged from r1622596, hive/trunk/ql/src/test/queries/clientpositive/uber_reduce.q
hive/branches/tez/ql/src/test/results/clientpositive/uber_reduce.q.out
- copied unchanged from r1622596, hive/trunk/ql/src/test/results/clientpositive/uber_reduce.q.out
Removed:
hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java
hive/branches/tez/service/src/test/org/apache/hive/service/cli/TestScratchDir.java
Modified:
hive/branches/tez/ (props changed)
hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
hive/branches/tez/itests/src/test/resources/testconfiguration.properties
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
hive/branches/tez/service/src/java/org/apache/hive/service/cli/CLIService.java
hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
hive/branches/tez/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
hive/branches/tez/service/src/java/org/apache/hive/service/server/HiveServer2.java
Propchange: hive/branches/tez/
------------------------------------------------------------------------------
Merged /hive/trunk:r1622374-1622596
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java Fri Sep 5 02:32:20 2014
@@ -25,9 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
/**
- *
- * ServerUtils.
- *
+ * ServerUtils (specific to HiveServer version 1)
*/
public class ServerUtils {
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Sep 5 02:32:20 2014
@@ -206,11 +206,18 @@ public class HiveConf extends Configurat
PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo",
"Query plan format serialization between client and task nodes. \n" +
"Two supported values are : kryo and javaXML. Kryo is default."),
- SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-${system:user.name}", "Scratch space for Hive jobs"),
+ SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive",
+ "HDFS root scratch dir for Hive jobs which gets created with 777 permission. " +
+ "For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, " +
+ "with ${hive.scratch.dir.permission}."),
LOCALSCRATCHDIR("hive.exec.local.scratchdir",
"${system:java.io.tmpdir}" + File.separator + "${system:user.name}",
"Local scratch space for Hive jobs"),
- SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", ""),
+ DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
+ "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources",
+ "Temporary local directory for added resources in the remote file system."),
+ SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700",
+ "The permission for the user specific scratch directories that get created."),
SUBMITVIACHILD("hive.exec.submitviachild", false, ""),
SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true,
"Determines whether local tasks (typically mapjoin hashtable generation phase) runs in \n" +
@@ -274,9 +281,6 @@ public class HiveConf extends Configurat
"Maximum number of dynamic partitions allowed to be created in each mapper/reducer node."),
MAXCREATEDFILES("hive.exec.max.created.files", 100000L,
"Maximum number of HDFS files created by all mappers/reducers in a MapReduce job."),
- DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
- "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources",
- "Temporary local directory for added resources in the remote file system."),
DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__",
"The default partition name in case the dynamic partition column value is null/empty string or any other values that cannot be escaped. \n" +
"This value must not contain any special character used in HDFS URI (e.g., ':', '%', '/' etc). \n" +
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Fri Sep 5 02:32:20 2014
@@ -1422,7 +1422,7 @@ sub run
$testStatuses->{$testName} = $failedStr;
}
- $msg= "$msg at " . time . "\n";
+ $msg .= "\nEnding test $testName at " . $endTime ."\n";
#print $msg;
print $log $msg;
$duration = $endTime - $beginTime;
@@ -1435,6 +1435,7 @@ sub run
if ($@) {
$msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n";
+ $msg .= "Ending test $testName at " . time ."\n";
#print $msg;
print $log $msg;
$testStatuses->{$testName} = $abortedStr;
Modified: hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java Fri Sep 5 02:32:20 2014
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
@@ -57,6 +56,7 @@ public class MiniHS2 extends AbstractHiv
private static final AtomicLong hs2Counter = new AtomicLong();
private MiniMrShim mr;
private MiniDFSShim dfs;
+ private FileSystem localFS;
private boolean useMiniMR = false;
private boolean useMiniKdc = false;
private final String serverPrincipal;
@@ -137,6 +137,10 @@ public class MiniHS2 extends AbstractHiv
this.dfs = dfs;
}
+ public FileSystem getLocalFS() {
+ return localFS;
+ }
+
public boolean isUseMiniMR() {
return useMiniMR;
}
@@ -157,7 +161,8 @@ public class MiniHS2 extends AbstractHiv
this.serverPrincipal = serverPrincipal;
this.serverKeytab = serverKeytab;
this.isMetastoreRemote = isMetastoreRemote;
- baseDir = Files.createTempDir();
+ baseDir = Files.createTempDir();
+ localFS = FileSystem.getLocal(hiveConf);
FileSystem fs;
if (useMiniMR) {
dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null);
@@ -371,7 +376,7 @@ public class MiniHS2 extends AbstractHiv
getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal());
sessionConf.put("principal", serverPrincipal);
}
- */
+ */
sessionHandle = hs2Client.openSession("foo", "bar", sessionConf);
} catch (Exception e) {
// service not started yet
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java Fri Sep 5 02:32:20 2014
@@ -31,7 +31,9 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
@@ -41,255 +43,357 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
- public class TestJdbcWithMiniHS2 {
- private static MiniHS2 miniHS2 = null;
- private static Path dataFilePath;
-
- private Connection hs2Conn = null;
-
- @BeforeClass
- public static void beforeTest() throws Exception {
- Class.forName(MiniHS2.getJdbcDriverName());
- HiveConf conf = new HiveConf();
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- miniHS2 = new MiniHS2(conf);
- String dataFileDir = conf.get("test.data.files").replace('\\', '/')
- .replace("c:", "");
- dataFilePath = new Path(dataFileDir, "kv1.txt");
- Map<String, String> confOverlay = new HashMap<String, String>();
- miniHS2.start(confOverlay);
- }
+public class TestJdbcWithMiniHS2 {
+ private static MiniHS2 miniHS2 = null;
+ private static Path dataFilePath;
+
+ private Connection hs2Conn = null;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ HiveConf conf = new HiveConf();
+ conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ miniHS2 = new MiniHS2(conf);
+ String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+ .replace("c:", "");
+ dataFilePath = new Path(dataFileDir, "kv1.txt");
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ miniHS2.start(confOverlay);
+ }
- @Before
- public void setUp() throws Exception {
- hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
- }
+ @Before
+ public void setUp() throws Exception {
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
+ }
- private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException {
- Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
- conn.createStatement().execute("set hive.support.concurrency = false");
- return conn;
- }
+ private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException {
+ Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
+ conn.createStatement().execute("set hive.support.concurrency = false");
+ return conn;
+ }
- @After
- public void tearDown() throws Exception {
- hs2Conn.close();
+ @After
+ public void tearDown() throws Exception {
+ hs2Conn.close();
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
}
+ }
+
+ @Test
+ public void testConnection() throws Exception {
+ String tableName = "testTab1";
+ Statement stmt = hs2Conn.createStatement();
+
+ // create table
+ stmt.execute("DROP TABLE IF EXISTS " + tableName);
+ stmt.execute("CREATE TABLE " + tableName
+ + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'");
+
+ // load data
+ stmt.execute("load data local inpath '"
+ + dataFilePath.toString() + "' into table " + tableName);
+
+ ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
+ assertTrue(res.next());
+ assertEquals("val_238", res.getString(2));
+ res.close();
+ stmt.close();
+ }
+
- @AfterClass
- public static void afterTest() throws Exception {
- if (miniHS2.isStarted())
- miniHS2.stop();
+ /** This test is to connect to any database without using the command "Use <<DB>>"
+ * 1)connect to default database.
+ * 2) Create a new DB test_default.
+ * 3) Connect to test_default database.
+ * 4) Connect and create table under test_default_test.
+ * 5) Connect and display all tables.
+ * 6) Connect to default database and shouldn't find table test_default_test.
+ * 7) Connect and drop test_default_test.
+ * 8) drop test_default database.
+ */
+
+ @Test
+ public void testURIDatabaseName() throws Exception{
+
+ String jdbcUri = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default"));
+
+ hs2Conn= getConnection(jdbcUri+"default", System.getProperty("user.name"),"bar");
+ String dbName="test_connection_non_default_db";
+ String tableInNonDefaultSchema="table_in_non_default_schema";
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("create database if not exists "+dbName);
+ stmt.close();
+ hs2Conn.close();
+
+ hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)");
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ ResultSet res = stmt.executeQuery("show tables");
+ boolean testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
+ }
+ assertTrue("table name "+tableInNonDefaultSchema
+ + " found in SHOW TABLES result set", testTableExists);
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ res = stmt.executeQuery("show tables");
+ testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
}
- @Test
- public void testConnection() throws Exception {
- String tableName = "testTab1";
- Statement stmt = hs2Conn.createStatement();
-
- // create table
- stmt.execute("DROP TABLE IF EXISTS " + tableName);
- stmt.execute("CREATE TABLE " + tableName
- + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'");
-
- // load data
- stmt.execute("load data local inpath '"
- + dataFilePath.toString() + "' into table " + tableName);
-
- ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
- assertTrue(res.next());
- assertEquals("val_238", res.getString(2));
- res.close();
- stmt.close();
+ assertFalse("table name "+tableInNonDefaultSchema
+ + " NOT found in SHOW TABLES result set", testTableExists);
+ stmt.close();
+ hs2Conn .close();
+
+ hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ stmt.execute("set hive.support.concurrency = false");
+ res = stmt.executeQuery("show tables");
+
+ stmt.execute(" drop table if exists table_in_non_default_schema");
+ expected = stmt.execute("DROP DATABASE "+ dbName);
+ stmt.close();
+
+ hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
+ stmt = hs2Conn .createStatement();
+ res = stmt.executeQuery("show tables");
+ testTableExists = false;
+ while (res.next()) {
+ assertNotNull("table name is null in result set", res.getString(1));
+ if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
+ testTableExists = true;
+ }
}
+ // test URI with no dbName
+ hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+
+ hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar");
+ verifyCurrentDB("default", hs2Conn);
+ hs2Conn.close();
+ }
- /** This test is to connect to any database without using the command "Use <<DB>>"
- * 1)connect to default database.
- * 2) Create a new DB test_default.
- * 3) Connect to test_default database.
- * 4) Connect and create table under test_default_test.
- * 5) Connect and display all tables.
- * 6) Connect to default database and shouldn't find table test_default_test.
- * 7) Connect and drop test_default_test.
- * 8) drop test_default database.
+ @Test
+ public void testConnectionSchemaAPIs() throws Exception {
+ String db1 = "DB1";
+ /**
+ * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6.
+ * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs
*/
+ HiveConnection hiveConn = (HiveConnection)hs2Conn;
- @Test
- public void testURIDatabaseName() throws Exception{
+ assertEquals("default", hiveConn.getSchema());
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
+ stmt.execute("CREATE DATABASE " + db1);
+ assertEquals("default", hiveConn.getSchema());
+
+ stmt.execute("USE " + db1);
+ assertEquals(db1, hiveConn.getSchema());
+
+ stmt.execute("USE default");
+ assertEquals("default", hiveConn.getSchema());
+
+ hiveConn.setSchema(db1);
+ assertEquals(db1, hiveConn.getSchema());
+ hiveConn.setSchema("default");
+ assertEquals("default", hiveConn.getSchema());
+
+ assertTrue(hiveConn.getCatalog().isEmpty());
+ hiveConn.setCatalog("foo");
+ assertTrue(hiveConn.getCatalog().isEmpty());
+ }
- String jdbcUri = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default"));
+ /**
+ * verify that the current db is the one expected. first create table as <db>.tab and then
+ * describe that table to check if <db> is the current database
+ * @param expectedDbName
+ * @param hs2Conn
+ * @throws Exception
+ */
+ private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception {
+ String verifyTab = "miniHS2DbVerificationTable";
+ Statement stmt = hs2Conn.createStatement();
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)");
+ stmt.execute("DESCRIBE " + verifyTab);
+ stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
+ stmt.close();
+ }
- hs2Conn= getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- String dbName="test_connection_non_default_db";
- String tableInNonDefaultSchema="table_in_non_default_schema";
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("create database if not exists "+dbName);
- stmt.close();
- hs2Conn.close();
-
- hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)");
- stmt.close();
- hs2Conn .close();
-
- hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- ResultSet res = stmt.executeQuery("show tables");
- boolean testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
- assertTrue("table name "+tableInNonDefaultSchema
- + " found in SHOW TABLES result set", testTableExists);
- stmt.close();
- hs2Conn .close();
-
- hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- res = stmt.executeQuery("show tables");
- testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
-
- assertFalse("table name "+tableInNonDefaultSchema
- + " NOT found in SHOW TABLES result set", testTableExists);
- stmt.close();
- hs2Conn .close();
-
- hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- stmt.execute("set hive.support.concurrency = false");
- res = stmt.executeQuery("show tables");
-
- stmt.execute(" drop table if exists table_in_non_default_schema");
- expected = stmt.execute("DROP DATABASE "+ dbName);
- stmt.close();
-
- hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar");
- stmt = hs2Conn .createStatement();
- res = stmt.executeQuery("show tables");
- testTableExists = false;
- while (res.next()) {
- assertNotNull("table name is null in result set", res.getString(1));
- if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) {
- testTableExists = true;
- }
- }
-
- // test URI with no dbName
- hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
-
- hs2Conn = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
-
- hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar");
- verifyCurrentDB("default", hs2Conn);
- hs2Conn.close();
- }
-
- @Test
- public void testConnectionSchemaAPIs() throws Exception {
- String db1 = "DB1";
- /**
- * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6.
- * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs
- */
- HiveConnection hiveConn = (HiveConnection)hs2Conn;
-
- assertEquals("default", hiveConn.getSchema());
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE");
- stmt.execute("CREATE DATABASE " + db1);
- assertEquals("default", hiveConn.getSchema());
-
- stmt.execute("USE " + db1);
- assertEquals(db1, hiveConn.getSchema());
-
- stmt.execute("USE default");
- assertEquals("default", hiveConn.getSchema());
-
- hiveConn.setSchema(db1);
- assertEquals(db1, hiveConn.getSchema());
- hiveConn.setSchema("default");
- assertEquals("default", hiveConn.getSchema());
-
- assertTrue(hiveConn.getCatalog().isEmpty());
- hiveConn.setCatalog("foo");
- assertTrue(hiveConn.getCatalog().isEmpty());
- }
-
- /**
- * verify that the current db is the one expected. first create table as <db>.tab and then
- * describe that table to check if <db> is the current database
- * @param expectedDbName
- * @param hs2Conn
- * @throws Exception
- */
- private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception {
- String verifyTab = "miniHS2DbVerificationTable";
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
- stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)");
- stmt.execute("DESCRIBE " + verifyTab);
- stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab);
- stmt.close();
- }
-
- /**
- * This method tests whether while creating a new connection, the config
- * variables specified in the JDBC URI are properly set for the connection.
- * This is a test for HiveConnection#configureConnection.
- *
- * @throws Exception
- */
- @Test
- public void testNewConnectionConfiguration() throws Exception {
-
- // Set some conf parameters
- String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
- + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
- // Set some conf vars
- String hiveVar = "stab=salesTable;icol=customerID";
- String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
-
- // Open a new connection with these conf & vars
- Connection con1 = DriverManager.getConnection(jdbcUri);
-
- // Execute "set" command and retrieve values for the conf & vars specified
- // above
- // Assert values retrieved
- Statement stmt = con1.createStatement();
-
- // Verify that the property has been properly set while creating the
- // connection above
- verifyConfProperty(stmt, "hive.cli.print.header", "true");
- verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
- verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
- verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
- "15");
- verifyConfProperty(stmt, "stab", "salesTable");
- verifyConfProperty(stmt, "icol", "customerID");
- con1.close();
- }
-
- private void verifyConfProperty(Statement stmt, String property,
- String expectedValue) throws Exception {
- ResultSet res = stmt.executeQuery("set " + property);
- while (res.next()) {
- String resultValues[] = res.getString(1).split("=");
- assertEquals(resultValues[1], expectedValue);
- }
- }
+ /**
+ * This method tests whether while creating a new connection, the config
+ * variables specified in the JDBC URI are properly set for the connection.
+ * This is a test for HiveConnection#configureConnection.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testNewConnectionConfiguration() throws Exception {
+
+ // Set some conf parameters
+ String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
+ + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
+ // Set some conf vars
+ String hiveVar = "stab=salesTable;icol=customerID";
+ String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
+
+ // Open a new connection with these conf & vars
+ Connection con1 = DriverManager.getConnection(jdbcUri);
+
+ // Execute "set" command and retrieve values for the conf & vars specified
+ // above
+ // Assert values retrieved
+ Statement stmt = con1.createStatement();
+
+ // Verify that the property has been properly set while creating the
+ // connection above
+ verifyConfProperty(stmt, "hive.cli.print.header", "true");
+ verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
+ verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
+ verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
+ "15");
+ verifyConfProperty(stmt, "stab", "salesTable");
+ verifyConfProperty(stmt, "icol", "customerID");
+ con1.close();
+ }
+
+ private void verifyConfProperty(Statement stmt, String property,
+ String expectedValue) throws Exception {
+ ResultSet res = stmt.executeQuery("set " + property);
+ while (res.next()) {
+ String resultValues[] = res.getString(1).split("=");
+ assertEquals(resultValues[1], expectedValue);
+ }
+ }
+
+ /**
+ * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded resources (which is also local).
+ * 1. Test with doAs=false: open a new JDBC session and verify the presence of directories/permissions
+ * 2. Test with doAs=true: open a new JDBC session and verify the presence of directories/permissions
+ * @throws Exception
+ */
+ @Test
+ public void testScratchDirs() throws Exception {
+ // Stop HiveServer2
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ HiveConf conf = new HiveConf();
+ String userName;
+ Path scratchDirPath;
+ // 1. Test with doAs=false
+ conf.setBoolean("hive.server2.enable.doAs", false);
+ // Set a custom prefix for hdfs scratch dir path
+ conf.set("hive.exec.scratchdir", "/tmp/hs2");
+ // Set a scratch dir permission
+ String fsPermissionStr = "700";
+ conf.set("hive.scratch.dir.permission", fsPermissionStr);
+ // Start an instance of HiveServer2 which uses miniMR
+ miniHS2 = new MiniHS2(conf);
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ miniHS2.start(confOverlay);
+ userName = System.getProperty("user.name");
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password");
+ // FS
+ FileSystem fs = miniHS2.getLocalFS();
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // 2. Test with doAs=true
+ // Restart HiveServer2 with doAs=true
+ if (miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ conf.setBoolean("hive.server2.enable.doAs", true);
+ // Start HS2
+ miniHS2 = new MiniHS2(conf);
+ miniHS2.start(confOverlay);
+ // Test for user "neo"
+ userName = "neo";
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Test for user "trinity"
+ userName = "trinity";
+ hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one");
+
+ // Verify scratch dir paths and permission
+ // HDFS scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) + "/" + userName);
+ verifyScratchDir(conf, fs, scratchDirPath, userName, false);
+
+ // Local scratch dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+
+ // Downloaded resources dir
+ scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ verifyScratchDir(conf, fs, scratchDirPath, userName, true);
+ }
+
+ private void verifyScratchDir(HiveConf conf, FileSystem fs, Path scratchDirPath,
+ String userName, boolean isLocal) throws Exception {
+ String dirType = isLocal ? "Local" : "DFS";
+ FsPermission expectedFSPermission = new FsPermission(HiveConf.getVar(conf,
+ HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+ assertTrue("The expected " + dirType + " scratch dir does not exist for the user: " +
+ userName, fs.exists(scratchDirPath));
+ if (fs.exists(scratchDirPath) && !isLocal) {
+ assertEquals("DFS scratch dir permissions don't match", expectedFSPermission,
+ fs.getFileStatus(scratchDirPath).getPermission());
+ }
}
+}
\ No newline at end of file
Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java (original)
+++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java Fri Sep 5 02:32:20 2014
@@ -19,16 +19,16 @@
package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-import java.util.Map;
-import java.util.HashMap;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -48,11 +48,11 @@ public class TestJdbcWithMiniMr {
public static final String TEST_TAG = "miniHS2.miniMr.tag";
public static final String TEST_TAG_VALUE = "miniHS2.miniMr.value";
public static class MiniMrTestSessionHook implements HiveSessionHook {
- @Override
- public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
- sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
- }
- }
+ @Override
+ public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
+ sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
+ }
+ }
private static MiniHS2 miniHS2 = null;
private static HiveConf conf;
@@ -93,7 +93,7 @@ public class TestJdbcWithMiniMr {
@Before
public void setUp() throws Exception {
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
+ System.getProperty("user.name"), "bar");
stmt = hs2Conn.createStatement();
stmt.execute("USE " + dbName);
}
@@ -225,7 +225,7 @@ public class TestJdbcWithMiniMr {
String queryStr = "SELECT * FROM " + tempTableName +
" where value = '" + resultVal + "'";
verifyResult(queryStr, resultVal, 2);
-
+
// A second connection should not be able to see the table
Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
System.getProperty("user.name"), "bar");
Modified: hive/branches/tez/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/src/test/resources/testconfiguration.properties?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/tez/itests/src/test/resources/testconfiguration.properties Fri Sep 5 02:32:20 2014
@@ -44,6 +44,7 @@ minimr.query.files=auto_sortmerge_join_1
stats_counter_partitioned.q,\
temp_table_external.q,\
truncate_column_buckets.q,\
+ uber_reduce.q,\
udf_using.q
minitez.query.files.shared=alter_merge_2_orc.q,\
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Context.java Fri Sep 5 02:32:20 2014
@@ -18,6 +18,18 @@
package org.apache.hadoop.hive.ql;
+import java.io.DataInput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -30,7 +42,6 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
@@ -41,18 +52,6 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.StringUtils;
-import java.io.DataInput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
/**
* Context for Semantic Analyzers. Usage: not reusable - construct a new one for
* each query should call clear() at end of use to remove temporary folders
@@ -191,7 +190,7 @@ public class Context {
* @param scratchDir path of tmp directory
*/
private Path getScratchDir(String scheme, String authority,
- boolean mkdir, String scratchDir) {
+ boolean mkdir, String scratchDir) {
String fileSystem = scheme + ":" + authority;
Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
@@ -203,11 +202,11 @@ public class Context {
try {
FileSystem fs = dirPath.getFileSystem(conf);
dirPath = new Path(fs.makeQualified(dirPath).toString());
- FsPermission fsPermission = new FsPermission(Short.parseShort(scratchDirPermission.trim(), 8));
+ FsPermission fsPermission = new FsPermission(scratchDirPermission);
- if (!Utilities.createDirsWithPermission(conf, dirPath, fsPermission)) {
+ if (!fs.mkdirs(dirPath, fsPermission)) {
throw new RuntimeException("Cannot make directory: "
- + dirPath.toString());
+ + dirPath.toString());
}
if (isHDFSCleanup) {
fs.deleteOnExit(dirPath);
@@ -233,7 +232,7 @@ public class Context {
FileSystem fs = FileSystem.getLocal(conf);
URI uri = fs.getUri();
return getScratchDir(uri.getScheme(), uri.getAuthority(),
- mkdir, localScratchDir);
+ mkdir, localScratchDir);
} catch (IOException e) {
throw new RuntimeException (e);
}
@@ -257,7 +256,7 @@ public class Context {
URI uri = dir.toUri();
Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
- !explain, uri.getPath());
+ !explain, uri.getPath());
LOG.info("New scratch dir is " + newScratchDir);
return newScratchDir;
} catch (IOException e) {
@@ -270,7 +269,7 @@ public class Context {
private Path getExternalScratchDir(URI extURI) {
return getScratchDir(extURI.getScheme(), extURI.getAuthority(),
- !explain, nonLocalScratchPath.toUri().getPath());
+ !explain, nonLocalScratchPath.toUri().getPath());
}
/**
@@ -283,7 +282,7 @@ public class Context {
p.getFileSystem(conf).delete(p, true);
} catch (Exception e) {
LOG.warn("Error Removing Scratch: "
- + StringUtils.stringifyException(e));
+ + StringUtils.stringifyException(e));
}
}
fsScratchDirs.clear();
@@ -305,7 +304,7 @@ public class Context {
*/
public boolean isMRTmpFileURI(String uriStr) {
return (uriStr.indexOf(executionId) != -1) &&
- (uriStr.indexOf(MR_PREFIX) != -1);
+ (uriStr.indexOf(MR_PREFIX) != -1);
}
/**
@@ -315,7 +314,7 @@ public class Context {
*/
public Path getMRTmpPath() {
return new Path(getMRScratchDir(), MR_PREFIX +
- nextPathId());
+ nextPathId());
}
/**
@@ -343,7 +342,7 @@ public class Context {
return getExtTmpPathRelTo(path.getParent());
}
return new Path(getExternalScratchDir(extURI), EXT_PREFIX +
- nextPathId());
+ nextPathId());
}
/**
@@ -353,8 +352,8 @@ public class Context {
*/
public Path getExtTmpPathRelTo(Path path) {
URI uri = path.toUri();
- return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain,
- uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId());
+ return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain,
+ uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId());
}
/**
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Fri Sep 5 02:32:20 2014
@@ -4017,6 +4017,9 @@ public class DDLTask extends Task<DDLWor
tbl.setDbName(names[0]);
tbl.setTableName(names[1]);
+ // using old table object, hence reset the owner to current user for new table.
+ tbl.setOwner(SessionState.getUserFromAuthenticator());
+
if (crtTbl.getLocation() != null) {
tbl.setDataLocation(new Path(crtTbl.getLocation()));
} else {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Sep 5 02:32:20 2014
@@ -92,7 +92,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.HiveInterruptCallback;
import org.apache.hadoop.hive.common.HiveInterruptUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
@@ -326,7 +325,9 @@ public final class Utilities {
assert path != null;
if (!gWorkMap.containsKey(path)) {
Path localPath;
- if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
+ if (conf.getBoolean("mapreduce.task.uberized", false) && name.equals(REDUCE_PLAN_NAME)) {
+ localPath = new Path(name);
+ } else if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
localPath = path;
} else {
localPath = new Path(name);
@@ -3355,7 +3356,6 @@ public final class Utilities {
private static void createTmpDirs(Configuration conf,
List<Operator<? extends OperatorDesc>> ops) throws IOException {
- FsPermission fsPermission = new FsPermission((short)00777);
while (!ops.isEmpty()) {
Operator<? extends OperatorDesc> op = ops.remove(0);
@@ -3365,7 +3365,8 @@ public final class Utilities {
if (tempDir != null) {
Path tempPath = Utilities.toTempPath(tempDir);
- createDirsWithPermission(conf, tempPath, fsPermission);
+ FileSystem fs = tempPath.getFileSystem(conf);
+ fs.mkdirs(tempPath);
}
}
@@ -3501,76 +3502,6 @@ public final class Utilities {
}
/**
- * @param conf the configuration used to derive the filesystem to create the path
- * @param mkdir the path to be created
- * @param fsPermission ignored if it is hive server session and doAs is enabled
- * @return true if successfully created the directory else false
- * @throws IOException if hdfs experiences any error conditions
- */
- public static boolean createDirsWithPermission(Configuration conf, Path mkdir,
- FsPermission fsPermission) throws IOException {
-
- boolean recursive = false;
- if (SessionState.get() != null) {
- recursive = SessionState.get().isHiveServerQuery() &&
- conf.getBoolean(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname,
- HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.defaultBoolVal);
- // we reset the permission in case of hive server and doAs enabled because
- // currently scratch directory uses /tmp/hive-hive as the scratch directory.
- // However, with doAs enabled, the first user to create this directory would
- // own the directory and subsequent users cannot access the scratch directory.
- // The right fix is to have scratch dir per user.
- fsPermission = new FsPermission((short)00777);
- }
-
- // if we made it so far without exception we are good!
- return createDirsWithPermission(conf, mkdir, fsPermission, recursive);
- }
-
- private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask,
- String origUmask, FileSystem fs) throws IOException {
- if (unsetUmask) {
- if (origUmask != null) {
- conf.set(FsPermission.UMASK_LABEL, origUmask);
- } else {
- conf.unset(FsPermission.UMASK_LABEL);
- }
- }
-
- fs.close();
- }
-
- public static boolean createDirsWithPermission(Configuration conf, Path mkdirPath,
- FsPermission fsPermission, boolean recursive) throws IOException {
- String origUmask = null;
- LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission + " recursive " +
- recursive);
-
- if (recursive) {
- origUmask = conf.get(FsPermission.UMASK_LABEL);
- // this umask is required because by default the hdfs mask is 022 resulting in
- // all parents getting the fsPermission & !(022) permission instead of fsPermission
- conf.set(FsPermission.UMASK_LABEL, "000");
- }
-
- FileSystem fs = ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf);
- boolean retval = false;
- try {
- retval = fs.mkdirs(mkdirPath, fsPermission);
- resetConfAndCloseFS(conf, recursive, origUmask, fs);
- } catch (IOException ioe) {
- try {
- resetConfAndCloseFS(conf, recursive, origUmask, fs);
- }
- catch (IOException e) {
- // do nothing - double failure
- }
- }
- return retval;
- }
-
-
- /**
* Convert path to qualified path.
*
* @param conf
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java Fri Sep 5 02:32:20 2014
@@ -43,7 +43,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -308,11 +308,11 @@ public class TezSessionState {
throws IOException {
// tez needs its own scratch dir (per session)
- Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR);
+ Path tezDir = new Path(SessionState.get().getHdfsScratchDirURIString(), TEZ_DIR);
tezDir = new Path(tezDir, sessionId);
FileSystem fs = tezDir.getFileSystem(conf);
- FsPermission fsPermission = new FsPermission((short)00777);
- Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true);
+ FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION));
+ fs.mkdirs(tezDir, fsPermission);
// Make sure the path is normalized (we expect validation to pass since we just created it).
tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath();
// don't keep the directory around on non-clean exit
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java Fri Sep 5 02:32:20 2014
@@ -52,6 +52,7 @@ public interface AcidOutputFormat<K exte
private int bucket;
private PrintStream dummyStream = null;
private boolean oldStyle = false;
+ private int recIdCol = -1; // Column the record identifier is in, -1 indicates no record id
/**
* Create the options object.
@@ -164,6 +165,16 @@ public interface AcidOutputFormat<K exte
}
/**
+ * Which column the row id field is in.
+ * @param recIdCol
+ * @return this
+ */
+ public Options recordIdColumn(int recIdCol) {
+ this.recIdCol = recIdCol;
+ return this;
+ }
+
+ /**
* Temporary switch while we are in development that replaces the
* implementation with a dummy one that just prints to stream.
* @param stream the stream to print to
@@ -214,6 +225,10 @@ public interface AcidOutputFormat<K exte
return bucket;
}
+ public int getRecordIdColumn() {
+ return recIdCol;
+ }
+
public PrintStream getDummyStream() {
return dummyStream;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java Fri Sep 5 02:32:20 2014
@@ -40,26 +40,17 @@ public interface RecordUpdater {
/**
* Update an old record with a new set of values.
* @param currentTransaction the current transaction id
- * @param originalTransaction the row's original transaction id
- * @param rowId the original row id
* @param row the new values for the row
* @throws IOException
*/
- void update(long currentTransaction,
- long originalTransaction,
- long rowId,
- Object row) throws IOException;
+ void update(long currentTransaction, Object row) throws IOException;
/**
* Delete a row from the table.
* @param currentTransaction the current transaction id
- * @param originalTransaction the rows original transaction id
- * @param rowId the original row id
* @throws IOException
*/
- void delete(long currentTransaction,
- long originalTransaction,
- long rowId) throws IOException;
+ void delete(long currentTransaction, Object row) throws IOException;
/**
* Flush the current set of rows to the underlying file system, so that
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java Fri Sep 5 02:32:20 2014
@@ -211,18 +211,14 @@ public class OrcOutputFormat extends Fil
}
@Override
- public void update(long currentTransaction, long originalTransaction,
- long rowId, Object row) throws IOException {
+ public void update(long currentTransaction, Object row) throws IOException {
out.println("update " + path + " currTxn: " + currentTransaction +
- " origTxn: " + originalTransaction + " row: " + rowId + " obj: " +
- stringifyObject(row, inspector));
+ " obj: " + stringifyObject(row, inspector));
}
@Override
- public void delete(long currentTransaction, long originalTransaction,
- long rowId) throws IOException {
- out.println("delete " + path + " currTxn: " + currentTransaction +
- " origTxn: " + originalTransaction + " row: " + rowId);
+ public void delete(long currentTransaction, Object row) throws IOException {
+ out.println("delete " + path + " currTxn: " + currentTransaction + " obj: " + row);
}
@Override
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java Fri Sep 5 02:32:20 2014
@@ -31,18 +31,18 @@ import org.apache.hadoop.hive.ql.io.Reco
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
import java.util.ArrayList;
import java.util.List;
@@ -92,6 +92,14 @@ public class OrcRecordUpdater implements
// because that is monotonically increasing to give new unique row ids.
private long rowCountDelta = 0;
private final KeyIndexBuilder indexBuilder = new KeyIndexBuilder();
+ private StructField recIdField = null; // field to look for the record identifier in
+ private StructField rowIdField = null; // field inside recId to look for row id in
+ private StructField originalTxnField = null; // field inside recId to look for original txn in
+ private StructObjectInspector rowInspector; // OI for the original row
+ private StructObjectInspector recIdInspector; // OI for the record identifier struct
+ private LongObjectInspector rowIdInspector; // OI for the long row id inside the recordIdentifier
+ private LongObjectInspector origTxnInspector; // OI for the original txn inside the record
+ // identifer
static class AcidStats {
long inserts;
@@ -179,7 +187,7 @@ public class OrcRecordUpdater implements
* @param rowInspector the row's object inspector
* @return an object inspector for the event stream
*/
- static ObjectInspector createEventSchema(ObjectInspector rowInspector) {
+ static StructObjectInspector createEventSchema(ObjectInspector rowInspector) {
List<StructField> fields = new ArrayList<StructField>();
fields.add(new OrcStruct.Field("operation",
PrimitiveObjectInspectorFactory.writableIntObjectInspector, OPERATION));
@@ -237,7 +245,9 @@ public class OrcRecordUpdater implements
writerOptions.bufferSize(DELTA_BUFFER_SIZE);
writerOptions.stripeSize(DELTA_STRIPE_SIZE);
}
- writerOptions.inspector(createEventSchema(options.getInspector()));
+ rowInspector = (StructObjectInspector)options.getInspector();
+ writerOptions.inspector(createEventSchema(findRecId(options.getInspector(),
+ options.getRecordIdColumn())));
this.writer = OrcFile.createWriter(this.path, writerOptions);
item = new OrcStruct(FIELDS);
item.setFieldValue(OPERATION, operation);
@@ -247,14 +257,50 @@ public class OrcRecordUpdater implements
item.setFieldValue(ROW_ID, rowId);
}
- private void addEvent(int operation, long currentTransaction,
- long originalTransaction, long rowId,
- Object row) throws IOException {
+ // Find the record identifier column (if there) and return a possibly new ObjectInspector that
+ // will strain out the record id for the underlying writer.
+ private ObjectInspector findRecId(ObjectInspector inspector, int rowIdColNum) {
+ if (!(inspector instanceof StructObjectInspector)) {
+ throw new RuntimeException("Serious problem, expected a StructObjectInspector, but got a " +
+ inspector.getClass().getName());
+ }
+ if (rowIdColNum < 0) {
+ return inspector;
+ } else {
+ RecIdStrippingObjectInspector newInspector =
+ new RecIdStrippingObjectInspector(inspector, rowIdColNum);
+ recIdField = newInspector.getRecId();
+ List<? extends StructField> fields =
+ ((StructObjectInspector) recIdField.getFieldObjectInspector()).getAllStructFieldRefs();
+ // Go by position, not field name, as field names aren't guaranteed. The order of fields
+ // in RecordIdentifier is transactionId, bucketId, rowId
+ originalTxnField = fields.get(0);
+ origTxnInspector = (LongObjectInspector)originalTxnField.getFieldObjectInspector();
+ rowIdField = fields.get(2);
+ rowIdInspector = (LongObjectInspector)rowIdField.getFieldObjectInspector();
+
+
+ recIdInspector = (StructObjectInspector) recIdField.getFieldObjectInspector();
+ return newInspector;
+ }
+ }
+
+ private void addEvent(int operation, long currentTransaction, long rowId, Object row)
+ throws IOException {
this.operation.set(operation);
this.currentTransaction.set(currentTransaction);
- this.originalTransaction.set(originalTransaction);
+ // If this is an insert, originalTransaction should be set to this transaction. If not,
+ // it will be reset by the following if anyway.
+ long originalTransaction = currentTransaction;
+ if (operation == DELETE_OPERATION || operation == UPDATE_OPERATION) {
+ Object rowIdValue = rowInspector.getStructFieldData(row, recIdField);
+ originalTransaction = origTxnInspector.get(
+ recIdInspector.getStructFieldData(rowIdValue, originalTxnField));
+ rowId = rowIdInspector.get(recIdInspector.getStructFieldData(rowIdValue, rowIdField));
+ }
this.rowId.set(rowId);
- item.setFieldValue(OrcRecordUpdater.ROW, row);
+ this.originalTransaction.set(originalTransaction);
+ item.setFieldValue(OrcRecordUpdater.ROW, (operation == DELETE_OPERATION ? null : row));
indexBuilder.addKey(operation, originalTransaction, bucket.get(), rowId);
writer.addRow(item);
}
@@ -264,30 +310,26 @@ public class OrcRecordUpdater implements
if (this.currentTransaction.get() != currentTransaction) {
insertedRows = 0;
}
- addEvent(INSERT_OPERATION, currentTransaction, currentTransaction,
- insertedRows++, row);
+ addEvent(INSERT_OPERATION, currentTransaction, insertedRows++, row);
rowCountDelta++;
}
@Override
- public void update(long currentTransaction, long originalTransaction,
- long rowId, Object row) throws IOException {
+ public void update(long currentTransaction, Object row) throws IOException {
if (this.currentTransaction.get() != currentTransaction) {
insertedRows = 0;
}
- addEvent(UPDATE_OPERATION, currentTransaction, originalTransaction, rowId,
- row);
+ addEvent(UPDATE_OPERATION, currentTransaction, -1L, row);
}
@Override
- public void delete(long currentTransaction, long originalTransaction,
- long rowId) throws IOException {
+ public void delete(long currentTransaction, Object row) throws IOException {
if (this.currentTransaction.get() != currentTransaction) {
insertedRows = 0;
}
- addEvent(DELETE_OPERATION, currentTransaction, originalTransaction, rowId,
- null);
+ addEvent(DELETE_OPERATION, currentTransaction, -1, row);
rowCountDelta--;
+
}
@Override
@@ -311,7 +353,7 @@ public class OrcRecordUpdater implements
fs.delete(path, false);
}
} else {
- writer.close();
+ if (writer != null) writer.close();
}
if (flushLengths != null) {
flushLengths.close();
@@ -406,4 +448,67 @@ public class OrcRecordUpdater implements
lastRowId = rowId;
}
}
+
+ /**
+ * An ObjectInspector that will strip out the record identifier so that the underlying writer
+ * doesn't see it.
+ */
+ private static class RecIdStrippingObjectInspector extends StructObjectInspector {
+ private StructObjectInspector wrapped;
+ List<StructField> fields;
+ StructField recId;
+
+ RecIdStrippingObjectInspector(ObjectInspector oi, int rowIdColNum) {
+ if (!(oi instanceof StructObjectInspector)) {
+ throw new RuntimeException("Serious problem, expected a StructObjectInspector, " +
+ "but got a " + oi.getClass().getName());
+ }
+ wrapped = (StructObjectInspector)oi;
+ List<? extends StructField> wrappedFields = wrapped.getAllStructFieldRefs();
+ fields = new ArrayList<StructField>(wrapped.getAllStructFieldRefs().size());
+ for (int i = 0; i < wrappedFields.size(); i++) {
+ if (i == rowIdColNum) {
+ recId = wrappedFields.get(i);
+ } else {
+ fields.add(wrappedFields.get(i));
+ }
+ }
+ }
+
+ @Override
+ public List<? extends StructField> getAllStructFieldRefs() {
+ return fields;
+ }
+
+ @Override
+ public StructField getStructFieldRef(String fieldName) {
+ return wrapped.getStructFieldRef(fieldName);
+ }
+
+ @Override
+ public Object getStructFieldData(Object data, StructField fieldRef) {
+ // For performance don't check that that the fieldRef isn't recId everytime,
+ // just assume that the caller used getAllStructFieldRefs and thus doesn't have that fieldRef
+ return wrapped.getStructFieldData(data, fieldRef);
+ }
+
+ @Override
+ public List<Object> getStructFieldsDataAsList(Object data) {
+ return wrapped.getStructFieldsDataAsList(data);
+ }
+
+ @Override
+ public String getTypeName() {
+ return wrapped.getTypeName();
+ }
+
+ @Override
+ public Category getCategory() {
+ return wrapped.getCategory();
+ }
+
+ StructField getRecId() {
+ return recId;
+ }
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Sep 5 02:32:20 2014
@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.ql.MapRedStats;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,6 +68,7 @@ import org.apache.hadoop.hive.ql.securit
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl;
import org.apache.hadoop.hive.ql.util.DosToUnix;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.base.Preconditions;
@@ -208,6 +208,8 @@ public class SessionState {
*/
private Path localSessionPath;
+ private String hdfsScratchDirURIString;
+
/**
* Get the lineage state stored in this session.
*
@@ -344,36 +346,39 @@ public class SessionState {
setCurrentSessionState(startSs);
- if(startSs.hiveHist == null){
+ if (startSs.hiveHist == null){
if (startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) {
startSs.hiveHist = new HiveHistoryImpl(startSs);
- }else {
- //Hive history is disabled, create a no-op proxy
+ } else {
+ // Hive history is disabled, create a no-op proxy
startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy();
}
}
- if (startSs.getTmpOutputFile() == null) {
- // set temp file containing results to be sent to HiveClient
- try {
- startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
// Get the following out of the way when you start the session these take a
// while and should be done when we start up.
try {
- //Hive object instance should be created with a copy of the conf object. If the conf is
+ // Hive object instance should be created with a copy of the conf object. If the conf is
// shared with SessionState, other parts of the code might update the config, but
// Hive.get(HiveConf) would not recognize the case when it needs refreshing
Hive.get(new HiveConf(startSs.conf)).getMSC();
- ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
+ UserGroupInformation sessionUGI = ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
FileSystem.get(startSs.conf);
- startSs.createSessionPaths(startSs.conf);
+
+ // Create scratch dirs for this session
+ startSs.createSessionDirs(sessionUGI.getShortUserName());
+
+ // Set temp file containing results to be sent to HiveClient
+ if (startSs.getTmpOutputFile() == null) {
+ try {
+ startSs.setTmpOutputFile(createTempFile(startSs.getConf()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
} catch (Exception e) {
- // catch-all due to some exec time dependencies on session state
+ // Catch-all due to some exec time dependencies on session state
// that would cause ClassNoFoundException otherwise
throw new RuntimeException(e);
}
@@ -396,6 +401,88 @@ public class SessionState {
return startSs;
}
+ /**
+ * Create dirs & session paths for this session:
+ * 1. HDFS scratch dir
+ * 2. Local scratch dir
+ * 3. Local downloaded resource dir
+ * 4. HDFS session path
+ * 5. Local session path
+ * 6. HDFS temp table space
+ * @param userName
+ * @throws IOException
+ */
+ private void createSessionDirs(String userName) throws IOException {
+ HiveConf conf = getConf();
+ // First create the root scratch dir on hdfs (if it doesn't already exist) and make it writable
+ Path rootHDFSDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR));
+ String rootHDFSDirPermission = "777";
+ createPath(conf, rootHDFSDirPath, rootHDFSDirPermission, false, false);
+ // Now create session specific dirs
+ String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ Path path;
+ // 1. HDFS scratch dir
+ path = new Path(rootHDFSDirPath, userName);
+ hdfsScratchDirURIString = path.toUri().toString();
+ createPath(conf, path, scratchDirPermission, false, false);
+ // 2. Local scratch dir
+ path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR));
+ createPath(conf, path, scratchDirPermission, true, false);
+ // 3. Download resources dir
+ path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ createPath(conf, path, scratchDirPermission, true, false);
+ // Finally, create session paths for this session
+ // Local & non-local tmp location is configurable. however it is the same across
+ // all external file systems
+ String sessionId = getSessionId();
+ // 4. HDFS session path
+ hdfsSessionPath = new Path(hdfsScratchDirURIString, sessionId);
+ createPath(conf, hdfsSessionPath, scratchDirPermission, false, true);
+ conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
+ // 5. Local session path
+ localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);
+ createPath(conf, localSessionPath, scratchDirPermission, true, true);
+ conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
+ // 6. HDFS temp table space
+ hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
+ createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false, true);
+ conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
+ }
+
+ /**
+ * Create a given path if it doesn't exist.
+ *
+ * @param conf
+ * @param pathString
+ * @param permission
+ * @param isLocal
+ * @param isCleanUp
+ * @return
+ * @throws IOException
+ */
+ private void createPath(HiveConf conf, Path path, String permission, boolean isLocal,
+ boolean isCleanUp) throws IOException {
+ FsPermission fsPermission = new FsPermission(permission);
+ FileSystem fs;
+ if (isLocal) {
+ fs = FileSystem.getLocal(conf);
+ } else {
+ fs = path.getFileSystem(conf);
+ }
+ if (!fs.exists(path)) {
+ fs.mkdirs(path, fsPermission);
+ String dirType = isLocal ? "local" : "HDFS";
+ LOG.info("Created " + dirType + " directory: " + path.toString());
+ }
+ if (isCleanUp) {
+ fs.deleteOnExit(path);
+ }
+ }
+
+ public String getHdfsScratchDirURIString() {
+ return hdfsScratchDirURIString;
+ }
+
public static Path getLocalSessionPath(Configuration conf) {
SessionState ss = SessionState.get();
if (ss == null) {
@@ -448,43 +535,6 @@ public class SessionState {
}
}
- private void createSessionPaths(Configuration conf) throws IOException {
-
- String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
- String sessionId = getSessionId();
-
- // local & non-local tmp location is configurable. however it is the same across
- // all external file systems
- hdfsSessionPath =
- new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
- sessionId);
- createPath(conf, hdfsSessionPath, scratchDirPermission);
- conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
-
- localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR),
- sessionId);
- createPath(conf, localSessionPath, scratchDirPermission);
- conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
- hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
- createPath(conf, hdfsTmpTableSpace, scratchDirPermission);
- conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
- }
-
- private void createPath(Configuration conf, Path p, String perm) throws IOException {
- FileSystem fs = p.getFileSystem(conf);
- p = new Path(fs.makeQualified(p).toString());
- FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(), 8));
-
- if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) {
- throw new IOException("Cannot create directory: "
- + p.toString());
- }
-
- // best effort to clean up if we don't shut down properly
- fs.deleteOnExit(p);
- }
-
-
/**
* Setup authentication and authorization plugins for this session.
*/
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java Fri Sep 5 02:32:20 2014
@@ -20,15 +20,12 @@ package org.apache.hadoop.hive.ql.exec;
import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension;
-import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import junit.framework.TestCase;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -39,7 +36,6 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.JobConf;
-import org.junit.Test;
public class TestUtilities extends TestCase {
@@ -77,9 +73,9 @@ public class TestUtilities extends TestC
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
children.add(constant);
ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo,
- new GenericUDFFromUtcTimestamp(), children);
+ new GenericUDFFromUtcTimestamp(), children);
assertEquals(desc.getExprString(), Utilities.deserializeExpression(
- Utilities.serializeExpression(desc)).getExprString());
+ Utilities.serializeExpression(desc)).getExprString());
}
public void testgetDbTableName() throws HiveException{
@@ -109,23 +105,4 @@ public class TestUtilities extends TestC
assertEquals("Invalid table name " + tablename, ex.getMessage());
}
}
-
- @Test
- public void testFSUmaskReset() throws Exception {
- // ensure that FS Umask is not reset (HIVE-7001)
- checkFSUMaskReset(true);
- checkFSUMaskReset(false);
- }
-
- private void checkFSUMaskReset(boolean recursiveArg) throws IllegalArgumentException, IOException {
- final String FS_MASK_VAL = "055";
- HiveConf conf = new HiveConf();
- String dir = System.getProperty("test.tmp.dir") + "/testUtilitiesUMaskReset";
- conf.set(FsPermission.UMASK_LABEL, FS_MASK_VAL);
- Utilities.createDirsWithPermission(conf, new Path(dir), new FsPermission((short) 00777),
- recursiveArg);
- assertEquals(conf.get(FsPermission.UMASK_LABEL), FS_MASK_VAL);
- }
-
-
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java Fri Sep 5 02:32:20 2014
@@ -56,6 +56,7 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
public class TestOrcRawRecordMerger {
@@ -454,9 +455,16 @@ public class TestOrcRawRecordMerger {
static class MyRow {
Text col1;
+ RecordIdentifier ROW__ID;
+
MyRow(String val) {
col1 = new Text(val);
}
+
+ MyRow(String val, long rowId, long origTxn, int bucket) {
+ col1 = new Text(val);
+ ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
+ }
}
static String getValue(OrcStruct event) {
@@ -533,12 +541,12 @@ public class TestOrcRawRecordMerger {
// write a delta
ru = of.getRecordUpdater(root, options.writingBase(false)
- .minimumTransactionId(200).maximumTransactionId(200));
- ru.update(200, 0, 0, new MyRow("update 1"));
- ru.update(200, 0, 2, new MyRow("update 2"));
- ru.update(200, 0, 3, new MyRow("update 3"));
- ru.delete(200, 0, 7);
- ru.delete(200, 0, 8);
+ .minimumTransactionId(200).maximumTransactionId(200).recordIdColumn(1));
+ ru.update(200, new MyRow("update 1", 0, 0, BUCKET));
+ ru.update(200, new MyRow("update 2", 2, 0, BUCKET));
+ ru.update(200, new MyRow("update 3", 3, 0, BUCKET));
+ ru.delete(200, new MyRow("", 7, 0, BUCKET));
+ ru.delete(200, new MyRow("", 8, 0, BUCKET));
ru.close(false);
ValidTxnList txnList = new ValidTxnListImpl("200:");
@@ -607,13 +615,13 @@ public class TestOrcRawRecordMerger {
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
- assertEquals(null, OrcRecordUpdater.getRow(event));
+ assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
- assertEquals(null, OrcRecordUpdater.getRow(event));
+ assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
@@ -693,7 +701,7 @@ public class TestOrcRawRecordMerger {
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
- assertEquals(null, OrcRecordUpdater.getRow(event));
+ assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
@@ -705,8 +713,7 @@ public class TestOrcRawRecordMerger {
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
- assertEquals(null, OrcRecordUpdater.getRow(event));
-
+ assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
OrcRecordUpdater.getOperation(event));
@@ -747,6 +754,7 @@ public class TestOrcRawRecordMerger {
Text mytext;
float myfloat;
double mydouble;
+ RecordIdentifier ROW__ID;
BigRow(int myint, long mylong, String mytext, float myfloat, double mydouble) {
this.myint = myint;
@@ -754,6 +762,21 @@ public class TestOrcRawRecordMerger {
this.mytext = new Text(mytext);
this.myfloat = myfloat;
this.mydouble = mydouble;
+ ROW__ID = null;
+ }
+
+ BigRow(int myint, long mylong, String mytext, float myfloat, double mydouble,
+ long rowId, long origTxn, int bucket) {
+ this.myint = myint;
+ this.mylong = mylong;
+ this.mytext = new Text(mytext);
+ this.myfloat = myfloat;
+ this.mydouble = mydouble;
+ ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
+ }
+
+ BigRow(long rowId, long origTxn, int bucket) {
+ ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
}
}
@@ -802,16 +825,16 @@ public class TestOrcRawRecordMerger {
// write a delta
AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
.writingBase(false).minimumTransactionId(1).maximumTransactionId(1)
- .bucket(BUCKET).inspector(inspector).filesystem(fs);
+ .bucket(BUCKET).inspector(inspector).filesystem(fs).recordIdColumn(5);
RecordUpdater ru = of.getRecordUpdater(root, options);
values = new String[]{"0.0", null, null, "1.1", null, null, null,
"ignore.7"};
for(int i=0; i < values.length; ++i) {
if (values[i] != null) {
- ru.update(1, 0, i, new BigRow(i, i, values[i], i, i));
+ ru.update(1, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
}
}
- ru.delete(100, 0, 9);
+ ru.delete(100, new BigRow(9, 0, BUCKET));
ru.close(false);
// write a delta
@@ -820,10 +843,10 @@ public class TestOrcRawRecordMerger {
values = new String[]{null, null, "1.0", null, null, null, null, "3.1"};
for(int i=0; i < values.length; ++i) {
if (values[i] != null) {
- ru.update(2, 0, i, new BigRow(i, i, values[i], i, i));
+ ru.update(2, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
}
}
- ru.delete(100, 0, 8);
+ ru.delete(100, new BigRow(8, 0, BUCKET));
ru.close(false);
InputFormat inf = new OrcInputFormat();
@@ -902,16 +925,16 @@ public class TestOrcRawRecordMerger {
ru.close(false);
// write a delta
- options.writingBase(false).minimumTransactionId(1).maximumTransactionId(1);
+ options.writingBase(false).minimumTransactionId(1).maximumTransactionId(1).recordIdColumn(5);
ru = of.getRecordUpdater(root, options);
values = new String[]{"0.0", null, null, "1.1", null, null, null,
"ignore.7"};
for(int i=0; i < values.length; ++i) {
if (values[i] != null) {
- ru.update(1, 0, i, new BigRow(i, i, values[i], i, i));
+ ru.update(1, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
}
}
- ru.delete(100, 0, 9);
+ ru.delete(100, new BigRow(9, 0, BUCKET));
ru.close(false);
// write a delta
@@ -920,10 +943,10 @@ public class TestOrcRawRecordMerger {
values = new String[]{null, null, "1.0", null, null, null, null, "3.1"};
for(int i=0; i < values.length; ++i) {
if (values[i] != null) {
- ru.update(2, 0, i, new BigRow(i, i, values[i], i, i));
+ ru.update(2, new BigRow(i, i, values[i], i, i, i, 0, BUCKET));
}
}
- ru.delete(100, 0, 8);
+ ru.delete(100, new BigRow(8, 0, BUCKET));
ru.close(false);
InputFormat inf = new OrcInputFormat();
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java?rev=1622597&r1=1622596&r2=1622597&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java Fri Sep 5 02:32:20 2014
@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.RecordUpdater;
-import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.IntWritable;
@@ -37,6 +37,7 @@ import java.io.DataInputStream;
import java.io.File;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class TestOrcRecordUpdater {
@@ -64,9 +65,18 @@ public class TestOrcRecordUpdater {
static class MyRow {
Text field;
+ RecordIdentifier ROW__ID;
+
MyRow(String val) {
field = new Text(val);
+ ROW__ID = null;
+ }
+
+ MyRow(String val, long rowId, long origTxn, int bucket) {
+ field = new Text(val);
+ ROW__ID = new RecordIdentifier(origTxn, bucket, rowId);
}
+
}
@Test
@@ -178,17 +188,19 @@ public class TestOrcRecordUpdater {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(MyRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
+ int bucket = 20;
AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
.filesystem(fs)
- .bucket(20)
+ .bucket(bucket)
.writingBase(false)
.minimumTransactionId(100)
.maximumTransactionId(100)
.inspector(inspector)
- .reporter(Reporter.NULL);
+ .reporter(Reporter.NULL)
+ .recordIdColumn(1);
RecordUpdater updater = new OrcRecordUpdater(root, options);
- updater.update(100, 10, 30, new MyRow("update"));
- updater.delete(100, 40, 60);
+ updater.update(100, new MyRow("update", 30, 10, bucket));
+ updater.delete(100, new MyRow("", 60, 40, bucket));
assertEquals(-1L, updater.getStats().getRowCount());
updater.close(false);
Path bucketPath = AcidUtils.createFilename(root, options);
@@ -216,7 +228,7 @@ public class TestOrcRecordUpdater {
assertEquals(40, OrcRecordUpdater.getOriginalTransaction(row));
assertEquals(20, OrcRecordUpdater.getBucket(row));
assertEquals(60, OrcRecordUpdater.getRowId(row));
- assertEquals(null, OrcRecordUpdater.getRow(row));
+ assertNull(OrcRecordUpdater.getRow(row));
assertEquals(false, rows.hasNext());
}
}