You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/01/12 04:59:07 UTC
svn commit: r1230390 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: hashutosh
Date: Thu Jan 12 03:59:06 2012
New Revision: 1230390
URL: http://svn.apache.org/viewvc?rev=1230390&view=rev
Log:
HIVE-2589: Newly created partition should inherit properties from table (Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1230390&r1=1230389&r2=1230390&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jan 12 03:59:06 2012
@@ -119,6 +119,7 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
+ HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
};
/**
@@ -296,6 +297,7 @@ public class HiveConf extends Configurat
METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true),
METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"),
METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners", ""),
+ METASTORE_PART_INHERIT_TBL_PROPS("hive.metastore.partition.inherit.table.properties",""),
// CLI
CLIIGNOREERRORS("hive.cli.errors.ignore", false),
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1230390&r1=1230389&r2=1230390&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Jan 12 03:59:06 2012
@@ -26,14 +26,17 @@ import static org.apache.hadoop.hive.met
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Formatter;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.Timer;
import java.util.regex.Pattern;
@@ -1562,6 +1565,24 @@ public class HiveMetaStore extends Thrif
part.getParameters().get(Constants.DDL_TIME) == null) {
part.putToParameters(Constants.DDL_TIME, Long.toString(time));
}
+
+
+ Map<String,String> tblParams = tbl.getParameters();
+ String inheritProps = hiveConf.getVar(ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim();
+ // Default value is empty string in which case no properties will be inherited.
+ // * implies all properties needs to be inherited
+ Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(",")));
+ if(inheritKeys.contains("*")){
+ inheritKeys = tblParams.keySet();
+ }
+
+ for (String key : inheritKeys) {
+ String paramVal = tblParams.get(key);
+ if(null != paramVal){ // add the property only if it exists in table properties
+ part.putToParameters(key, paramVal);
+ }
+ }
+
success = ms.addPartition(part);
} finally {
Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q Thu Jan 12 03:59:06 2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=a,b;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q Thu Jan 12 03:59:06 2012
@@ -0,0 +1,3 @@
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q Thu Jan 12 03:59:06 2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=key1,*;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out Thu Jan 12 03:59:06 2012
@@ -0,0 +1,47 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ a myval
+ b yourval
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out Thu Jan 12 03:59:06 2012
@@ -0,0 +1,45 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out?rev=1230390&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out Thu Jan 12 03:59:06 2012
@@ -0,0 +1,48 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ a myval
+ b yourval
+ c noval
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1