You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2015/05/22 22:50:51 UTC
hive git commit: HIVE-10776 - Schema on insert for bucketed tables
throwing NullPointerException(Eugene Koifman, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 95929308b -> 0d93438a3
HIVE-10776 - Schema on insert for bucketed tables throwing NullPointerException(Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d93438a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d93438a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d93438a
Branch: refs/heads/master
Commit: 0d93438a3543cb64cbe2ebcdc21e5b40c1dd86e6
Parents: 9592930
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri May 22 13:50:40 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri May 22 13:50:40 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 +-
.../clientpositive/insert_into_with_schema2.q | 23 +++++
.../insert_into_with_schema2.q.out | 98 ++++++++++++++++++++
3 files changed, 126 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 4346f60..7f355e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3910,7 +3910,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @throws SemanticException
*/
private void handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
- RowResolver out_rwsch, RowResolver inputRR, QB qb,
+ RowResolver outputRR, RowResolver inputRR, QB qb,
ASTNode selExprList) throws SemanticException {
//(z,x)
List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query
@@ -3932,7 +3932,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, ColumnInfo> targetCol2ColumnInfo = new HashMap<String, ColumnInfo>();
int colListPos = 0;
for(String targetCol : targetTableSchema) {
- targetCol2ColumnInfo.put(targetCol, out_rwsch.getColumnInfos().get(colListPos));
+ targetCol2ColumnInfo.put(targetCol, outputRR.getColumnInfos().get(colListPos));
targetCol2Projection.put(targetCol, col_list.get(colListPos++));
}
Table target = qb.getMetaData().getDestTableForAlias(dest);
@@ -3976,16 +3976,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
t.setText("TOK_NULL");
ExprNodeDesc exp = genExprNodeDesc(new ASTNode(t), inputRR, tcCtx);
new_col_list.add(exp);
- final String tableAlias = "";//is this OK? this column doesn't come from any table
+ final String tableAlias = null;//this column doesn't come from any table
ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos),
exp.getWritableObjectInspector(), tableAlias, false);
newSchema.add(colInfo);
+ outputRR.addMappingOnly(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo);
}
colListPos++;
}
col_list.clear();
col_list.addAll(new_col_list);
- out_rwsch.setRowSchema(new RowSchema(newSchema));
+ outputRR.setRowSchema(new RowSchema(newSchema));
}
String recommendName(ExprNodeDesc exp, String colAlias) {
if (!colAlias.startsWith(autogenColAliasPrfxLbl)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
new file mode 100644
index 0000000..b7c6b58
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
@@ -0,0 +1,23 @@
+-- SORT_QUERY_RESULTS;
+
+set hive.enforce.bucketing=true;
+
+create table studenttab10k (age2 int);
+insert into studenttab10k values(1);
+
+create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets;
+
+insert into student_acid(age) select * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) select 3 g, * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) values(20, 2);
+
+insert into student_acid(age) values(22);
+
+select * from student_acid;
http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
new file mode 100644
index 0000000..a55a82f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
@@ -0,0 +1,98 @@
+PREHOOK: query: create table studenttab10k (age2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: create table studenttab10k (age2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: insert into studenttab10k values(1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: insert into studenttab10k values(1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@studenttab10k
+POSTHOOK: Lineage: studenttab10k.age2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@student_acid
+PREHOOK: query: insert into student_acid(age) select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 NULL
+PREHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade SIMPLE []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 3
+1 NULL
+PREHOOK: query: insert into student_acid(grade, age) values(20, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) values(20, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: insert into student_acid(age) values(22)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__3
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) values(22)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__3
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 3
+1 NULL
+2 20
+22 NULL