You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2012/01/23 20:59:04 UTC
svn commit: r1234958 - in /hive/branches/branch-0.8-r2:
common/src/java/org/apache/hadoop/hive/conf/ conf/
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/queries/clientpositive/
ql/src/test/resul...
Author: cws
Date: Mon Jan 23 19:59:03 2012
New Revision: 1234958
URL: http://svn.apache.org/viewvc?rev=1234958&view=rev
Log:
HIVE-2589. Newly created partition should inherit properties from table (Ashutosh Chauhan via cws)
Added:
hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
Modified:
hive/branches/branch-0.8-r2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/branch-0.8-r2/conf/hive-default.xml.template
hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/branch-0.8-r2/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Modified: hive/branches/branch-0.8-r2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1234958&r1=1234957&r2=1234958&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/branch-0.8-r2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Jan 23 19:59:03 2012
@@ -119,6 +119,7 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
+ HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
};
/**
@@ -296,6 +297,7 @@ public class HiveConf extends Configurat
METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true),
METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"),
METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners", ""),
+ METASTORE_PART_INHERIT_TBL_PROPS("hive.metastore.partition.inherit.table.properties",""),
// CLI
CLIIGNOREERRORS("hive.cli.errors.ignore", false),
Modified: hive/branches/branch-0.8-r2/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/conf/hive-default.xml.template?rev=1234958&r1=1234957&r2=1234958&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/conf/hive-default.xml.template (original)
+++ hive/branches/branch-0.8-r2/conf/hive-default.xml.template Mon Jan 23 19:59:03 2012
@@ -244,6 +244,12 @@
</property>
<property>
+ <name>hive.metastore.partition.inherit.table.properties</name>
+ <value></value>
+ <description>list of comma seperated keys occurring in table properties which will get inherited to newly created partitions. * implies all the keys will get inherited.</description>
+</property>
+
+<property>
<name>hive.metastore.end.function.listeners</name>
<value></value>
<description>list of comma separated listeners for the end of metastore functions.</description>
Modified: hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1234958&r1=1234957&r2=1234958&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Mon Jan 23 19:59:03 2012
@@ -26,14 +26,17 @@ import static org.apache.hadoop.hive.met
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Formatter;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.Timer;
import java.util.regex.Pattern;
@@ -1563,6 +1566,24 @@ public class HiveMetaStore extends Thrif
part.getParameters().get(Constants.DDL_TIME) == null) {
part.putToParameters(Constants.DDL_TIME, Long.toString(time));
}
+
+ // Inherit table properties into partition properties.
+ Map<String,String> tblParams = tbl.getParameters();
+ String inheritProps = hiveConf.getVar(ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim();
+ // Default value is empty string in which case no properties will be inherited.
+ // * implies all properties needs to be inherited
+ Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(",")));
+ if(inheritKeys.contains("*")){
+ inheritKeys = tblParams.keySet();
+ }
+
+ for (String key : inheritKeys) {
+ String paramVal = tblParams.get(key);
+ if(null != paramVal){ // add the property only if it exists in table properties
+ part.putToParameters(key, paramVal);
+ }
+ }
+
success = ms.addPartition(part);
} finally {
Modified: hive/branches/branch-0.8-r2/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1234958&r1=1234957&r2=1234958&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/branch-0.8-r2/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Jan 23 19:59:03 2012
@@ -357,7 +357,7 @@ public class QTestUtil {
db.dropDatabase(dbName);
}
}
- db.setCurrentDatabase(DEFAULT_DATABASE_NAME);
+ Hive.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
List<String> roleNames = db.getAllRoleNames();
for (String roleName : roleNames) {
Added: hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q Mon Jan 23 19:59:03 2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=a,b;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q Mon Jan 23 19:59:03 2012
@@ -0,0 +1,4 @@
+set hive.metastore.partition.inherit.table.properties="";
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q Mon Jan 23 19:59:03 2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=key1,*;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');
Added: hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out Mon Jan 23 19:59:03 2012
@@ -0,0 +1,48 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+CreateTime: Sun Jan 22 09:09:54 PST 2012
+LastAccessTime: UNKNOWN
+Protect Mode: None
+Location: pfile:/home/ashutosh/workspace/hive8/build/ql/test/data/warehouse/mytbl/c2=v1
+Partition Parameters:
+ a myval
+ b yourval
+ transient_lastDdlTime 1327252194
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Added: hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out Mon Jan 23 19:59:03 2012
@@ -0,0 +1,46 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+CreateTime: Sun Jan 22 09:12:05 PST 2012
+LastAccessTime: UNKNOWN
+Protect Mode: None
+Location: pfile:/home/ashutosh/workspace/hive8/build/ql/test/data/warehouse/mytbl/c2=v1
+Partition Parameters:
+ transient_lastDdlTime 1327252325
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Added: hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out?rev=1234958&view=auto
==============================================================================
--- hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out (added)
+++ hive/branches/branch-0.8-r2/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out Mon Jan 23 19:59:03 2012
@@ -0,0 +1,49 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+c1 tinyint None
+
+# Partition Information
+# col_name data_type comment
+
+c2 string None
+
+# Detailed Partition Information
+Partition Value: [v1]
+Database: default
+Table: mytbl
+CreateTime: Sun Jan 22 09:04:19 PST 2012
+LastAccessTime: UNKNOWN
+Protect Mode: None
+Location: pfile:/home/ashutosh/workspace/hive8/build/ql/test/data/warehouse/mytbl/c2=v1
+Partition Parameters:
+ a myval
+ b yourval
+ c noval
+ transient_lastDdlTime 1327251859
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1