You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2010/09/23 19:09:18 UTC
svn commit: r1000539 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientp...
Author: heyongqiang
Date: Thu Sep 23 17:09:17 2010
New Revision: 1000539
URL: http://svn.apache.org/viewvc?rev=1000539&view=rev
Log:
HIVE-1661. Default values for parameters (Siying Dong via He Yongqiang)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_default_prop.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/create_default_prop.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=1000539&r1=1000538&r2=1000539&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Sep 23 17:09:17 2010
@@ -77,6 +77,9 @@ Trunk - Unreleased
HIVE-1609. Support partition filtering in metastore
(Ajay Kidave via jvs)
+ HIVE-1661. Default values for parameters
+ (Siying Dong via He Yongqiang)
+
IMPROVEMENTS
HIVE-1394. Do not update transient_lastDdlTime if the partition is modified by a housekeeping
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1000539&r1=1000538&r2=1000539&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Sep 23 17:09:17 2010
@@ -151,6 +151,9 @@ public class HiveConf extends Configurat
METASTORE_INT_EXTRACTED("hive.metastore.archive.intermediate.extracted",
"_INTERMEDIATE_EXTRACTED"),
+ // Default parameters for creating tables
+ NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),
+
// CLI
CLIIGNOREERRORS("hive.cli.errors.ignore", false),
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=1000539&r1=1000538&r2=1000539&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Thu Sep 23 17:09:17 2010
@@ -712,4 +712,10 @@
<description>Whether writes to HBase should be forced to the write-ahead log. Disabling this improves HBase write performance at the risk of lost writes in case of a crash.</description>
</property>
+<property>
+ <name>hive.table.parameters.default</name>
+ <value></value>
+ <description>Default property values for newly created tables</description>
+</property>
+
</configuration>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1000539&r1=1000538&r2=1000539&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Thu Sep 23 17:09:17 2010
@@ -2282,6 +2282,11 @@ public class DDLTask extends Task<DDLWor
private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException {
// create the table
Table tbl = new Table(db.getCurrentDatabase(), crtTbl.getTableName());
+
+ if (crtTbl.getTblProps() != null) {
+ tbl.getTTable().getParameters().putAll(crtTbl.getTblProps());
+ }
+
if (crtTbl.getPartCols() != null) {
tbl.setPartCols(crtTbl.getPartCols());
}
@@ -2329,9 +2334,6 @@ public class DDLTask extends Task<DDLWor
tbl.setSerdeParam(m.getKey(), m.getValue());
}
}
- if (crtTbl.getTblProps() != null) {
- tbl.getTTable().getParameters().putAll(crtTbl.getTblProps());
- }
/*
* We use LazySimpleSerDe by default.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1000539&r1=1000538&r2=1000539&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Sep 23 17:09:17 2010
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.Context;
@@ -6644,6 +6645,34 @@ public class SemanticAnalyzer extends Ba
}
/**
+ * Add default properties for table property. If a default parameter exists
+ * in the tblProp, the value in tblProp will be kept.
+ * @param table property map
+ * @return Modified table property map
+ */
+ private Map<String, String> addDefaultProperties(Map<String, String> tblProp) {
+ Map<String, String> retValue;
+ if (tblProp == null) {
+ retValue = new HashMap<String, String>();
+ } else {
+ retValue = tblProp;
+ }
+ String paraString = HiveConf.getVar(conf, ConfVars.NEWTABLEDEFAULTPARA);
+ if (paraString != null && !paraString.isEmpty()) {
+ for (String keyValuePair: paraString.split(",")) {
+ String[] keyValue = keyValuePair.split("=", 2);
+ if (keyValue.length != 2) {
+ continue;
+ }
+ if (!retValue.containsKey(keyValue[0])) {
+ retValue.put(keyValue[0], keyValue[1]);
+ }
+ }
+ }
+ return retValue;
+ }
+
+ /**
* Analyze the create table command. If it is a regular create-table or
* create-table-like statements, we create a DDLWork and return true. If it is
* a create-table-as-select, we get the necessary info such as the SerDe and
@@ -6800,6 +6829,7 @@ public class SemanticAnalyzer extends Ba
switch (command_type) {
case CREATE_TABLE: // REGULAR CREATE TABLE DDL
+ tblProps = addDefaultProperties(tblProps);
crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols,
bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape,
rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment,
@@ -6829,6 +6859,8 @@ public class SemanticAnalyzer extends Ba
throw new SemanticException(e);
}
+ tblProps = addDefaultProperties(tblProps);
+
crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols,
bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape,
rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, storageFormat.inputFormat,
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_default_prop.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_default_prop.q?rev=1000539&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_default_prop.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_default_prop.q Thu Sep 23 17:09:17 2010
@@ -0,0 +1,10 @@
+set hive.table.parameters.default=p1=v1,P2=v21=v22=v23;
+CREATE TABLE table_p1 (a STRING);
+DESC EXTENDED table_p1;
+
+set hive.table.parameters.default=p3=v3;
+CREATE TABLE table_p2 LIKE table_p1;
+DESC EXTENDED table_p2;
+
+CREATE TABLE table_p3 AS SELECT * FROM table_p1;
+DESC EXTENDED table_p3;
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/create_default_prop.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/create_default_prop.q.out?rev=1000539&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/create_default_prop.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/create_default_prop.q.out Thu Sep 23 17:09:17 2010
@@ -0,0 +1,116 @@
+PREHOOK: query: CREATE TABLE table_p1 (a STRING)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE table_p1 (a STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table_p1
+PREHOOK: query: DESC EXTENDED table_p1
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESC EXTENDED table_p1
+POSTHOOK: type: DESCTABLE
+col_name data_type comment
+
+a string None
+
+# Detailed Table Information
+Database: default
+Owner: sdong
+CreateTime: Tue Sep 21 20:51:39 PDT 2010
+LastAccessTime: Wed Dec 31 16:00:00 PST 1969
+Protect Mode: None
+Retention: 0
+Location: pfile:/data/users/sdong/www/trunk/VENDOR.hive/trunk/build/ql/test/data/warehouse/table_p1
+Table Type: MANAGED_TABLE
+Table Parameters:
+ p1 v1
+ transient_lastDdlTime 1285127499
+ P2 v21=v22=v23
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+
+PREHOOK: query: CREATE TABLE table_p2 LIKE table_p1
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE table_p2 LIKE table_p1
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table_p2
+PREHOOK: query: DESC EXTENDED table_p2
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESC EXTENDED table_p2
+POSTHOOK: type: DESCTABLE
+col_name data_type comment
+
+a string None
+
+# Detailed Table Information
+Database: default
+Owner: sdong
+CreateTime: Tue Sep 21 20:51:39 PDT 2010
+LastAccessTime: Wed Dec 31 16:00:00 PST 1969
+Protect Mode: None
+Retention: 0
+Location: pfile:/data/users/sdong/www/trunk/VENDOR.hive/trunk/build/ql/test/data/warehouse/table_p2
+Table Type: MANAGED_TABLE
+Table Parameters:
+ EXTERNAL FALSE
+ p1 v1
+ transient_lastDdlTime 1285127499
+ P2 v21=v22=v23
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+
+PREHOOK: query: CREATE TABLE table_p3 AS SELECT * FROM table_p1
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: default@table_p1
+POSTHOOK: query: CREATE TABLE table_p3 AS SELECT * FROM table_p1
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: default@table_p1
+POSTHOOK: Output: default@table_p3
+PREHOOK: query: DESC EXTENDED table_p3
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESC EXTENDED table_p3
+POSTHOOK: type: DESCTABLE
+col_name data_type comment
+
+a string None
+
+# Detailed Table Information
+Database: default
+Owner: sdong
+CreateTime: Tue Sep 21 20:51:43 PDT 2010
+LastAccessTime: Wed Dec 31 16:00:00 PST 1969
+Protect Mode: None
+Retention: 0
+Location: pfile:/data/users/sdong/www/trunk/VENDOR.hive/trunk/build/ql/test/data/warehouse/table_p3
+Table Type: MANAGED_TABLE
+Table Parameters:
+ p3 v3
+ transient_lastDdlTime 1285127503
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+