You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/07/31 20:25:10 UTC
hive git commit: HIVE-19694 : Create Materialized View statement
should check for MV name conflicts before running MV's SQL statement. (Miklos
Gergely via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master a3ed7d6fb -> 3104d4756
HIVE-19694 : Create Materialized View statement should check for MV name conflicts before running MV's SQL statement. (Miklos Gergely via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3104d475
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3104d475
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3104d475
Branch: refs/heads/master
Commit: 3104d4756a6ff4c7853e4a17bbef13317770efd2
Parents: a3ed7d6
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Tue Jul 31 13:24:35 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Jul 31 13:24:35 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 4 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 113 ++++---------------
.../materialized_view_name_collusion.q | 10 ++
.../materialized_view_name_collusion.q.out | 19 ++++
4 files changed, 50 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index a70aea0..61396e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1008,7 +1008,6 @@ public class CalcitePlanner extends SemanticAnalyzer {
table.setMaterializedTable(true);
LOG.info(cteName + " will be materialized into " + location);
- cte.table = table;
cte.source = analyzer;
ctx.addMaterializedTable(cteName, table);
@@ -1567,8 +1566,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
- RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, inputRR, qb,
- selExprList);
+ RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, qb, selExprList);
ArrayList<String> columnNames = new ArrayList<String>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b28cf98..474c793 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -169,7 +169,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPos
import org.apache.hadoop.hive.ql.optimizer.lineage.Generator;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
-import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
@@ -271,7 +270,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -280,7 +278,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.math.IntMath;
import com.google.common.math.LongMath;
@@ -386,9 +383,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
CreateTableDesc tableDesc;
- /** Not thread-safe. */
- final ASTSearcher astSearcher = new ASTSearcher();
-
protected AnalyzeRewriteContext analyzeRewrite;
private WriteEntity acidAnalyzeTable;
@@ -1156,15 +1150,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return this.nameToSplitSample;
}
- /**
- * Convert a string to Text format and write its bytes in the same way TextOutputFormat would do.
- * This is needed to properly encode non-ascii characters.
- */
- private static void writeAsText(String text, FSDataOutputStream out) throws IOException {
- Text to = new Text(text);
- out.write(to.getBytes(), 0, to.getLength());
- }
-
private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException {
String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ?
HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT):
@@ -1323,7 +1308,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<CTEClause> parents = new ArrayList<CTEClause>();
// materialized
- Table table;
SemanticAnalyzer source;
List<Task<?>> getTasks() {
@@ -1432,7 +1416,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
table.setMaterializedTable(true);
LOG.info(cteName + " will be materialized into " + location);
- cte.table = table;
cte.source = analyzer;
ctx.addMaterializedTable(cteName, table);
@@ -1551,15 +1534,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return alias;
}
- private String extractLateralViewAlias(ASTNode lateralView) {
- // Lateral view AST has the following shape:
- // ^(TOK_LATERAL_VIEW
- // ^(TOK_SELECT ^(TOK_SELEXPR ^(TOK_FUNCTION Identifier params) identifier* tableAlias)))
- ASTNode selExpr = (ASTNode) lateralView.getChild(0).getChild(0);
- ASTNode astTableAlias = (ASTNode) Iterables.getLast(selExpr.getChildren());
- return astTableAlias.getChild(0).getText();
- }
-
/**
* Phase 1: (including, but not limited to):
*
@@ -1928,7 +1902,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
/**
* This is phase1 of supporting specifying schema in insert statement
* insert into foo(z,y) select a,b from bar;
- * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, RowResolver, QB, ASTNode)
+ * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, QB, ASTNode)
* @throws SemanticException
*/
private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase1Ctx ctx_1) throws SemanticException {
@@ -4615,7 +4589,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
selectStar = selectStar && exprList.getChildCount() == posn + 1;
- out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, inputRR, qb, selExprList);
+ out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, qb, selExprList);
ArrayList<String> columnNames = new ArrayList<String>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
@@ -4667,7 +4641,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- boolean defaultConstraintsFetch = true;
for (int i = 0; i < targetTableColNames.size(); i++) {
String f = targetTableColNames.get(i);
if(targetCol2Projection.containsKey(f)) {
@@ -4726,7 +4699,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @throws SemanticException
*/
public RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
- RowResolver outputRR, RowResolver inputRR, QB qb,
+ RowResolver outputRR, QB qb,
ASTNode selExprList) throws SemanticException {
//(z,x)
List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query
@@ -4816,17 +4789,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
/**
- * Convert exprNodeDesc array to Typeinfo array.
- */
- static ArrayList<TypeInfo> getTypeInfo(ArrayList<ExprNodeDesc> exprs) {
- ArrayList<TypeInfo> result = new ArrayList<TypeInfo>();
- for (ExprNodeDesc expr : exprs) {
- result.add(expr.getTypeInfo());
- }
- return result;
- }
-
- /**
* Convert exprNodeDesc array to ObjectInspector array.
*/
static ArrayList<ObjectInspector> getWritableObjectInspector(ArrayList<ExprNodeDesc> exprs) {
@@ -4838,18 +4800,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
/**
- * Convert exprNodeDesc array to Typeinfo array.
- */
- static ObjectInspector[] getStandardObjectInspector(ArrayList<TypeInfo> exprs) {
- ObjectInspector[] result = new ObjectInspector[exprs.size()];
- for (int i = 0; i < exprs.size(); i++) {
- result[i] = TypeInfoUtils
- .getStandardWritableObjectInspectorFromTypeInfo(exprs.get(i));
- }
- return result;
- }
-
- /**
* Returns the GenericUDAFEvaluator for the aggregation. This is called once
* for each GroupBy aggregation.
*/
@@ -5594,7 +5544,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<String> outputKeyColumnNames = new ArrayList<String>();
List<String> outputValueColumnNames = new ArrayList<String>();
- ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, dest,
+ ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs,
reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames,
colExprMap);
@@ -5662,7 +5612,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return rsOp;
}
- private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs, String dest,
+ private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs,
RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver,
List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap)
throws SemanticException {
@@ -5800,7 +5750,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<String> outputValueColumnNames = new ArrayList<String>();
List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
- ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, dest,
+ ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs,
reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames,
colExprMap);
@@ -6062,7 +6012,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI;
containsDistinctAggr = containsDistinctAggr || isDistinct;
- boolean isStar = value.getType() == HiveParser.TOK_FUNCTIONSTAR;
Mode amode = groupByDescModeToUDAFMode(mode, isDistinct);
GenericUDAFEvaluator genericUDAFEvaluator = genericUDAFEvaluators
.get(entry.getKey());
@@ -6268,15 +6217,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return curr;
}
- static ArrayList<GenericUDAFEvaluator> getUDAFEvaluators(
- ArrayList<AggregationDesc> aggs) {
- ArrayList<GenericUDAFEvaluator> result = new ArrayList<GenericUDAFEvaluator>();
- for (int i = 0; i < aggs.size(); i++) {
- result.add(aggs.get(i).getGenericUDAFEvaluator());
- }
- return result;
- }
-
/**
* Generate a Group-By plan using a 2 map-reduce jobs (5 operators will be
* inserted):
@@ -11825,19 +11765,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
this.qb = qb;
}
- boolean analyzeCreateTable(ASTNode child) throws SemanticException {
- if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) {
- // if it is not CTAS, we don't need to go further and just return
- if ((child = analyzeCreateTable(ast, qb, null)) == null) {
- return true;
- }
- } else {
- queryState.setCommandType(HiveOperation.QUERY);
- }
-
- return false;
- }
-
@Override
@SuppressWarnings("nls")
public void analyzeInternal(ASTNode ast) throws SemanticException {
@@ -11857,13 +11784,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* Planner specific stuff goes in here.
*/
static class PlannerContext {
- protected ASTNode child;
- protected Phase1Ctx ctx_1;
-
- void setParseTreeAttr(ASTNode child, Phase1Ctx ctx_1) {
- this.child = child;
- this.ctx_1 = ctx_1;
- }
void setCTASToken(ASTNode child) {
}
@@ -12201,8 +12121,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
getMetaData(qb, createVwDesc == null);
LOG.info("Completed getting MetaData in Semantic Analysis");
- plannerCtx.setParseTreeAttr(child, ctx_1);
-
return true;
}
@@ -13593,10 +13511,23 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
storageFormat.fillDefaultStorageFormat(false, isMaterialized);
- if (ifNotExists && orReplace){
+ if (ifNotExists && orReplace) {
throw new SemanticException("Can't combine IF NOT EXISTS and OR REPLACE.");
}
+ if (isMaterialized && !ifNotExists) {
+ // Verify that the table does not already exist
+ // dumpTable is only used to check the conflict for non-temporary tables
+ try {
+ Table dumpTable = db.newTable(dbDotTable);
+ if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.isExplainSkipExecution()) {
+ throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTable));
+ }
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ }
+
if (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW &&
ast.getChild(1).getType() == HiveParser.TOK_QUERY) {
isAlterViewAs = true;
@@ -13629,10 +13560,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return selectStmt;
}
- CreateViewDesc getCreateViewDesc() {
- return this.createVwDesc;
- }
-
// validate the (materialized) view statement
// check semantic conditions
private void validateCreateView()
http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q b/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q
new file mode 100644
index 0000000..567ac2a
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q
@@ -0,0 +1,10 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.enforce.bucketing=true;
+
+create table mvnc_basetable (a int, b varchar(256), c decimal(10,2));
+
+
+create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable;
+
+create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable;
http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out b/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out
new file mode 100644
index 0000000..9a2cb3e
--- /dev/null
+++ b/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out
@@ -0,0 +1,19 @@
+PREHOOK: query: create table mvnc_basetable (a int, b varchar(256), c decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mvnc_basetable
+POSTHOOK: query: create table mvnc_basetable (a int, b varchar(256), c decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mvnc_basetable
+PREHOOK: query: create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@mvnc_basetable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mvnc_mat_view
+POSTHOOK: query: create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@mvnc_basetable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mvnc_mat_view
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Table already exists: default.mvnc_mat_view