You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/12/22 05:49:07 UTC
hive git commit: HIVE-18315 - update tests use non-acid tables
(Eugene Koifman, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master bc2848a26 -> 3e930bfa3
HIVE-18315 - update tests use non-acid tables (Eugene Koifman, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3e930bfa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3e930bfa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3e930bfa
Branch: refs/heads/master
Commit: 3e930bfa3a870ea85e017b751a07e49dfed32f74
Parents: bc2848a
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Dec 21 21:48:52 2017 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Dec 21 21:48:52 2017 -0800
----------------------------------------------------------------------
.../hcatalog/mapreduce/HCatMapReduceTest.java | 1 +
.../hcatalog/pig/AbstractHCatLoaderTest.java | 2 +
.../hcatalog/pig/AbstractHCatStorerTest.java | 180 ++++++-------------
.../hive/hcatalog/pig/TestE2EScenarios.java | 10 +-
.../pig/TestHCatLoaderComplexSchema.java | 13 +-
.../hive/hcatalog/pig/TestHCatStorerMulti.java | 11 +-
.../hive/hcatalog/streaming/TestStreaming.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnLoadData.java | 16 +-
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 23 ++-
.../hadoop/hive/ql/TxnCommandsBaseForTests.java | 5 +-
.../hive/ql/lockmgr/TestDbTxnManager.java | 5 +
.../hive/ql/lockmgr/TestDbTxnManager2.java | 2 +-
12 files changed, 98 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
index ae56ff7..29a006c 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
@@ -217,6 +217,7 @@ public abstract class HCatMapReduceTest extends HCatBaseTest {
tableParams.put(hive_metastoreConstants.IS_IMMUTABLE,"true");
}
StatsSetupConst.setBasicStatsState(tableParams, StatsSetupConst.TRUE);
+ tableParams.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "false");
tbl.setParameters(tableParams);
client.createTable(tbl);
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java
index 59d2efb..124a5bc 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java
@@ -112,6 +112,8 @@ public abstract class AbstractHCatLoaderTest extends HCatBaseTest {
createTable = createTable + "partitioned by (" + partitionedBy + ") ";
}
createTable = createTable + "stored as " +storageFormat;
+ //HCat doesn't support transactional tables
+ createTable += " TBLPROPERTIES ('transactional'='false')";
executeStatementOnDriver(createTable, driver);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java
index 4f7cf2b..4e4638a 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java
@@ -417,14 +417,9 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testPartColsInData() throws IOException, CommandNeedRetryException {
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int", "b string", driver, storageFormat);
- driver.run("drop table junit_unparted");
- String createTable =
- "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
int LOOP_SIZE = 11;
String[] input = new String[LOOP_SIZE];
for (int i = 0; i < LOOP_SIZE; i++) {
@@ -456,15 +451,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testMultiPartColsInData() throws Exception {
- driver.run("drop table employee");
- String createTable =
- "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) "
- + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat;
-
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("employee", driver);
+ AbstractHCatLoaderTest.createTable("employee",
+ "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING",
+ "emp_country STRING , emp_state STRING", driver, storageFormat);
String[] inputData =
{ "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA",
@@ -512,13 +502,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreInPartiitonedTbl() throws Exception {
- driver.run("drop table junit_unparted");
- String createTable =
- "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int", "b string",
+ driver, storageFormat);
+
int LOOP_SIZE = 11;
String[] input = new String[LOOP_SIZE];
for (int i = 0; i < LOOP_SIZE; i++) {
@@ -553,14 +540,8 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testNoAlias() throws IOException, CommandNeedRetryException {
- driver.run("drop table junit_parted");
- String createTable =
- "create table junit_parted(a int, b string) partitioned by (ds string) stored as "
- + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_parted", driver);
+ AbstractHCatLoaderTest.createTable("junit_parted","a int, b string", "ds string", driver, storageFormat);
PigServer server = new PigServer(ExecType.LOCAL);
boolean errCaught = false;
try {
@@ -603,19 +584,13 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreMultiTables() throws IOException, CommandNeedRetryException {
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null,
+ driver, storageFormat);
- driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
- driver.run("drop table junit_unparted2");
- createTable = "create table junit_unparted2(a int, b string) stored as RCFILE";
- retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted2", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted2","a int, b string", null,
+ driver, "RCFILE");
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE * LOOP_SIZE];
@@ -660,13 +635,9 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException {
-
- driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null,
+ driver, storageFormat);
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE * LOOP_SIZE];
@@ -700,13 +671,9 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException {
-
- driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null,
+ driver, storageFormat);
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE * LOOP_SIZE];
@@ -741,12 +708,8 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testEmptyStore() throws IOException, CommandNeedRetryException {
- driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null, driver, storageFormat);
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE * LOOP_SIZE];
@@ -777,15 +740,11 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testBagNStruct() throws IOException, CommandNeedRetryException {
- driver.run("drop table junit_unparted");
- String createTable =
- "create table junit_unparted(b string,a struct<a1:int>, arr_of_struct array<string>, "
- + "arr_of_struct2 array<struct<s1:string,s2:string>>, arr_of_struct3 array<struct<s3:string>>) stored as "
- + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted",
+ "b string,a struct<a1:int>, arr_of_struct array<string>, " +
+ "arr_of_struct2 array<struct<s1:string,s2:string>>, arr_of_struct3 array<struct<s3:string>>",
+ null, driver, storageFormat);
String[] inputData =
new String[] { "zookeeper\t(2)\t{(pig)}\t{(pnuts,hdfs)}\t{(hadoop),(hcat)}",
@@ -823,15 +782,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException {
-
- driver.run("drop table junit_unparted");
- String createTable =
- "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as "
- + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted",
+ "a int, b float, c double, d bigint, e string, h boolean, f binary, g binary", null,
+ driver, storageFormat);
int i = 0;
String[] input = new String[3];
@@ -887,13 +841,9 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testStoreFuncSimple() throws IOException, CommandNeedRetryException {
-
- driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("junit_unparted", driver);
+ AbstractHCatLoaderTest.createTable("junit_unparted","a int, b string", null,
+ driver, storageFormat);
int LOOP_SIZE = 3;
String[] inputData = new String[LOOP_SIZE * LOOP_SIZE];
@@ -930,16 +880,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOException,
CommandNeedRetryException {
-
- driver.run("drop table if exists employee");
- String createTable =
- "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) "
- + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat;
-
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("employee", driver);
+ AbstractHCatLoaderTest.createTable("employee",
+ "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING",
+ "emp_country STRING , emp_state STRING", driver, storageFormat);
String[] inputData =
{ "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA",
@@ -970,16 +914,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOException,
CommandNeedRetryException {
-
- driver.run("drop table if exists employee");
- String createTable =
- "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) "
- + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat;
-
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
+ AbstractHCatLoaderTest.dropTable("employee", driver);
+ AbstractHCatLoaderTest.createTable("employee",
+ "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING",
+ "emp_country STRING , emp_state STRING", driver, storageFormat);
String[] inputData =
{ "111237\tKrishna\t01/01/1990\tM\tIN\tTN", "111238\tKalpana\t01/01/2000\tF\tIN\tKA",
@@ -1009,16 +947,11 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOException,
CommandNeedRetryException {
+ AbstractHCatLoaderTest.dropTable("employee", driver);
+ AbstractHCatLoaderTest.createTable("employee",
+ "emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING",
+ "emp_country STRING , emp_state STRING", driver, storageFormat);
- driver.run("drop table if exists employee");
- String createTable =
- "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) "
- + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat;
-
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
String[] inputData = {};
HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData);
@@ -1040,15 +973,10 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
@Test
public void testPartitionPublish() throws IOException, CommandNeedRetryException {
+ AbstractHCatLoaderTest.dropTable("ptn_fail", driver);
+ AbstractHCatLoaderTest.createTable("ptn_fail","a int, c string", "b string",
+ driver, storageFormat);
- driver.run("drop table ptn_fail");
- String createTable =
- "create table ptn_fail(a int, c string) partitioned by (b string) stored as "
- + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table.");
- }
int LOOP_SIZE = 11;
String[] input = new String[LOOP_SIZE];
@@ -1065,7 +993,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest {
server.executeBatch();
String query = "show partitions ptn_fail";
- retCode = driver.run(query).getResponseCode();
+ int retCode = driver.run(query).getResponseCode();
if (retCode != 0) {
throw new IOException("Error " + retCode + " running query " + query);
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
index 4a6c6a3..95a8164 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
@@ -112,15 +112,7 @@ public class TestE2EScenarios {
}
private void createTable(String tablename, String schema, String partitionedBy, String storageFormat) throws IOException, CommandNeedRetryException {
- String createTable;
- createTable = "create table " + tablename + "(" + schema + ") ";
- if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
- createTable = createTable + "partitioned by (" + partitionedBy + ") ";
- }
- if (storageFormat != null){
- createTable = createTable + "stored as " +storageFormat;
- }
- driverRun(createTable);
+ AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat);
}
private void driverRun(String cmd) throws IOException, CommandNeedRetryException {
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
index ea9cdda..5c9bae4 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
@@ -97,18 +97,7 @@ public class TestHCatLoaderComplexSchema {
}
private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
- String createTable;
- createTable = "create table " + tablename + "(" + schema + ") ";
- if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
- createTable = createTable + "partitioned by (" + partitionedBy + ") ";
- }
- createTable = createTable + "stored as " + storageFormat;
- LOG.info("Creating table:\n {}", createTable);
- CommandProcessorResponse result = driver.run(createTable);
- int retCode = result.getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + " " + result.getErrorMessage() + "]");
- }
+ AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat);
}
private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
index 40ea923..e2c9b2e 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
@@ -83,16 +83,7 @@ public class TestHCatStorerMulti {
}
private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
- String createTable;
- createTable = "create table " + tablename + "(" + schema + ") ";
- if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
- createTable = createTable + "partitioned by (" + partitionedBy + ") ";
- }
- createTable = createTable + "stored as " + storageFormat;
- int retCode = driver.run(createTable).getResponseCode();
- if (retCode != 0) {
- throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + "]");
- }
+ AbstractHCatLoaderTest.createTable(tablename, schema, partitionedBy, driver, storageFormat);
}
private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index dc8eee1..101ba1a 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -511,7 +511,7 @@ public class TestStreaming {
runDDL(driver, "use testBucketing3");
runDDL(driver, "create table " + tbl1 + " ( key1 string, data string ) clustered by ( key1 ) into "
- + bucketCount + " buckets stored as orc location " + tableLoc) ;
+ + bucketCount + " buckets stored as orc location " + tableLoc + " TBLPROPERTIES ('transactional'='false')") ;
runDDL(driver, "create table " + tbl2 + " ( key1 string, data string ) clustered by ( key1 ) into "
+ bucketCount + " buckets stored as orc location " + tableLoc2 + " TBLPROPERTIES ('transactional'='false')") ;
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index b0dcc7d..c4911bb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -92,7 +92,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver(
"create table T (a int, b int) stored as orc tblproperties('transactional'='true')");
//Tstage is just a simple way to generate test data
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
//this creates an ORC data file with correct schema under table root
runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'");
@@ -160,7 +160,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')");
runStatementOnDriver("insert into T values(0,2),(0,4)");
//Tstage is just a simple way to generate test data
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
//this creates an ORC data file with correct schema under table root
runStatementOnDriver("export table Tstage to '" + getWarehouseDir() +"/1'");
@@ -230,10 +230,10 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
private void loadDataNonAcid2AcidConversion(boolean isVectorized) throws Exception {
runStatementOnDriver("drop table if exists T");
runStatementOnDriver("drop table if exists Tstage");
- runStatementOnDriver("create table T (a int, b int) stored as orc");
+ runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='false')");
//per acid write to test nonAcid2acid conversion mixed with load data
runStatementOnDriver("insert into T values(0,2),(0,4)");
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
//make 2 more inserts so that we have 000000_0_copy_1, 000000_0_copy_2 files in export
//export works at file level so if you have copy_N in the table dir, you'll have those in output
@@ -307,7 +307,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("drop table if exists T");
runStatementOnDriver("drop table if exists Tstage");
runStatementOnDriver("create table T (a int, b int) partitioned by (p int) stored as orc tblproperties('transactional'='true')");
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(0,2),(0,4)");
runStatementOnDriver("export table Tstage to '" + getWarehouseDir() +"/1'");
@@ -361,7 +361,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("create table T (a int, b int) clustered by (a) into 2 buckets stored as orc tblproperties('transactional'='true')");
File createdFile= folder.newFile("myfile.txt");
FileUtils.writeStringToFile(createdFile, "hello world");
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
//this creates an ORC data file with correct schema under table root
runStatementOnDriver("insert into Tstage values(1,2),(3,4)");
CommandProcessorResponse cpr = runStatementOnDriverNegative("load data local inpath '" + getWarehouseDir() + "' into table T");
@@ -389,7 +389,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("drop table if exists Tstage");
runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')");
//Tstage is just a simple way to generate test data
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(5,5),(6,6)");
//this creates an ORC data file with correct schema under table root
runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'");
@@ -430,7 +430,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("drop table if exists Tstage");
runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')");
//Tstage is just a simple way to generate test data
- runStatementOnDriver("create table Tstage (a int, b int) stored as orc");
+ runStatementOnDriver("create table Tstage (a int, b int) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into Tstage values(5,5),(6,6)");
//this creates an ORC data file with correct schema under table root
runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'");
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index 780b97c..ac44779 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -67,7 +68,7 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
int[][] sourceVals1 = {{0,0,0},{3,3,3}};
int[][] sourceVals2 = {{1,1,1},{2,2,2}};
runStatementOnDriver("drop table if exists tmp");
- runStatementOnDriver("create table tmp (c1 integer, c2 integer, c3 integer) stored as orc");
+ runStatementOnDriver("create table tmp (c1 integer, c2 integer, c3 integer) stored as orc tblproperties('transactional'='false')");
runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
runStatementOnDriver("drop table if exists nobuckets");
@@ -187,7 +188,7 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values));
runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES ('transactional" +
- "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);
+ "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid in listener is too late
rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas2 order by ROW__ID");
String expected2[][] = {
{"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000017_0000017_0000/bucket_00000"},
@@ -531,7 +532,7 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
//this enables vectorization of ROW__ID
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ROW_IDENTIFIER_ENABLED, true);//HIVE-12631
runStatementOnDriver("drop table if exists T");
- runStatementOnDriver("create table T(a int, b int) stored as orc");
+ runStatementOnDriver("create table T(a int, b int) stored as orc tblproperties('transactional'='false')");
int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
runStatementOnDriver("insert into T(a, b) " + makeValuesClause(values));
//, 'transactional_properties'='default'
@@ -696,5 +697,21 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
map = hms.getPartitionColumnStatistics("default","T", partNames, colNames);
Assert.assertEquals("", 5, map.get(partNames.get(0)).get(0).getStatsData().getLongStats().getHighValue());
}
+ @Ignore("enable after HIVE-18294")
+ @Test
+ public void testDefault() throws Exception {
+ runStatementOnDriver("drop table if exists T");
+ runStatementOnDriver("create table T (a int, b int) stored as orc");
+ runStatementOnDriver("insert into T values(1,2),(3,4)");
+ String query = "select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b";
+ List<String> rs = runStatementOnDriver(query);
+ String[][] expected = {
+ //this proves data is written in Acid layout so T was made Acid
+ {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/delta_0000015_0000015_0000/bucket_00000"},
+ {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000015_0000015_0000/bucket_00000"}
+ };
+ checkExpected(rs, expected, "insert data");
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
index 32a2489..30bbb3f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -71,7 +72,7 @@ public abstract class TxnCommandsBaseForTests {
public void setUp() throws Exception {
setUpInternal();
}
- protected void setUpInternal() throws Exception {
+ void setUpInternal() throws Exception {
hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
@@ -100,7 +101,7 @@ public abstract class TxnCommandsBaseForTests {
runStatementOnDriver("create table " + Table.NONACIDORCTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
runStatementOnDriver("create temporary table " + Table.ACIDTBL2 + "(a int, b int, c int) clustered by (c) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
- runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + "(a int, b int) stored as orc");
+ runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + "(a int, b int) stored as orc TBLPROPERTIES ('transactional'='false')");
}
protected void dropTables() throws Exception {
for(TxnCommandsBaseForTests.Table t : TxnCommandsBaseForTests.Table.values()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
index 406bdea..e84f63d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
@@ -388,6 +388,11 @@ public class TestDbTxnManager {
@Test
public void concurrencyFalse() throws Exception {
HiveConf badConf = new HiveConf();
+ if(badConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY)) {
+ //TxnManagerFactory is a singleton, so if the default is true, it has already been
+ //created and won't throw
+ return;
+ }
badConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER,
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
badConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/3e930bfa/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index d309e3d..28131a0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -623,7 +623,7 @@ public class TestDbTxnManager2 {
CommandProcessorResponse cpr = null;
cpr = driver.run("create table acidPart(a int, b int) partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
- cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc");
+ cpr = driver.run("create table nonAcidPart(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='false')");
checkCmdOnDriver(cpr);
cpr = driver.compileAndRespond("insert into nonAcidPart partition(p) values(1,2,3)");