You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2014/09/20 16:52:10 UTC
svn commit: r1626449 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/
Author: gates
Date: Sat Sep 20 14:52:09 2014
New Revision: 1626449
URL: http://svn.apache.org/r1626449
Log:
HIVE-8149 hive.optimize.reducededuplication should be set to false for IUD ops (Alan Gates, reviewed by Eugene Koifman)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q
hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q
hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q
hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q
hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Sep 20 14:52:09 2014
@@ -6240,6 +6240,9 @@ public class SemanticAnalyzer extends Ba
// Check constraints on acid tables. This includes
// * no insert overwrites
// * no use of vectorization
+ // * turns off reduce deduplication optimization, as that sometimes breaks acid
+ // This method assumes you have already decided that this is an Acid write. Don't call it if
+ // that isn't true.
private void checkAcidConstraints(QB qb, TableDesc tableDesc) throws SemanticException {
String tableName = tableDesc.getTableName();
if (!qb.getParseInfo().isInsertIntoTable(tableName)) {
@@ -6250,6 +6253,9 @@ public class SemanticAnalyzer extends Ba
LOG.info("Turning off vectorization for acid write operation");
conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false);
}
+ LOG.info("Modifying config values for ACID write");
+ conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, false);
+ conf.setBoolVar(ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES, true);
}
/**
Modified: hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/acid_vectorization.q Sat Sep 20 14:52:09 2014
@@ -4,7 +4,6 @@ set hive.input.format=org.apache.hadoop.
set hive.enforce.bucketing=true;
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.vectorized.execution.enabled=true;
-set hive.mapred.supports.subdirectories=true;
CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC;
insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10;
Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.exec.reducers.max = 1;
create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_all_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_where_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/delete_whole_partition.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_update_delete.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q Sat Sep 20 14:52:09 2014
@@ -3,7 +3,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table ivdp(i int,
de decimal(5,2),
Modified: hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert_values_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.exec.dynamic.partition.mode=nonstrict;
create table acid_ivp(ti tinyint,
si smallint,
Modified: hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q Sat Sep 20 14:52:09 2014
@@ -3,7 +3,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_uami(i int,
de decimal(5,2),
Modified: hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_all_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;
Modified: hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q?rev=1626449&r1=1626448&r2=1626449&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/update_where_partitioned.q Sat Sep 20 14:52:09 2014
@@ -2,7 +2,6 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
set hive.enforce.bucketing=true;
-set hive.mapred.supports.subdirectories=true;
create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;