You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/02/07 19:12:45 UTC
[4/5] hive git commit: HIVE-16605: Enforce NOT NULL constraint
(Vineet Garg, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/c50ebb34/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b67a03f..9ccb4e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -52,6 +52,7 @@ import org.antlr.runtime.tree.TreeVisitorAction;
import org.antlr.runtime.tree.TreeWizard;
import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -127,6 +128,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -210,6 +212,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCardinalityViolation;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
@@ -574,7 +577,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn,
- (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1));
+ (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1));
// If this is a duplicate invocation of a function; don't add to WindowingSpec.
if ( wExprsInDest != null &&
wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
@@ -611,8 +614,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @throws SemanticException
*/
private void doPhase1GetAllAggregations(ASTNode expressionTree,
- HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns,
- ASTNode wndParent) throws SemanticException {
+ HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns,
+ ASTNode wndParent) throws SemanticException {
int exprTokenType = expressionTree.getToken().getType();
if(exprTokenType == HiveParser.TOK_SUBQUERY_EXPR) {
//since now we have scalar subqueries we can get subquery expression in having
@@ -792,9 +795,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
}
TableSample tabSample = new TableSample(
- unescapeIdentifier(sampleClause.getChild(0).getText()),
- unescapeIdentifier(sampleClause.getChild(1).getText()),
- sampleCols);
+ unescapeIdentifier(sampleClause.getChild(0).getText()),
+ unescapeIdentifier(sampleClause.getChild(1).getText()),
+ sampleCols);
qb.getParseInfo().setTabSample(alias, tabSample);
if (unparseTranslator.isEnabled()) {
for (ASTNode sampleCol : sampleCols) {
@@ -876,8 +879,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException {
String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ?
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT):
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT):
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
if (!inputFormat.equals(CombineHiveInputFormat.class.getName())) {
throw new SemanticException(generateErrorMessage((ASTNode) numerator,
message + " sampling is not supported in " + inputFormat));
@@ -1164,8 +1167,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
static private boolean isOuterJoinToken(ASTNode node) {
return (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN)
- || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN)
- || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN);
+ || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN)
+ || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN);
}
/**
@@ -1197,7 +1200,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
processPTF(qb, child);
PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child);
String inputAlias = ptfInvocationSpec == null ? null :
- ptfInvocationSpec.getFunction().getAlias();;
+ ptfInvocationSpec.getFunction().getAlias();;
if ( inputAlias == null ) {
throw new SemanticException(generateErrorMessage(child,
"PTF invocation in a Join must have an alias"));
@@ -1328,7 +1331,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
doPhase1GetColumnAliasesFromSelect(ast, qbp);
qbp.setAggregationExprsForClause(ctx_1.dest, aggregations);
qbp.setDistinctFuncExprsForClause(ctx_1.dest,
- doPhase1GetDistinctFuncExprs(aggregations));
+ doPhase1GetDistinctFuncExprs(aggregations));
break;
case HiveParser.TOK_WHERE:
@@ -1449,7 +1452,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
break;
case HiveParser.TOK_SORTBY:
- // Get the sort by aliases - these are aliased to the entries in the
+ // Get the sort by aliases - these are aliased to the entries in the
// select list
queryProperties.setHasSortBy(true);
qbp.setSortByExprForClause(ctx_1.dest, ast);
@@ -1645,17 +1648,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<String> targetColNames = new ArrayList<String>();
for(Node col : tabColName.getChildren()) {
assert ((ASTNode)col).getType() == HiveParser.Identifier :
- "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType();
+ "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType();
targetColNames.add(((ASTNode)col).getText().toLowerCase());
}
String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0),
- SessionState.get().getCurrentDatabase());
+ SessionState.get().getCurrentDatabase());
qbp.setDestSchemaForClause(ctx_1.dest, targetColNames);
Set<String> targetColumns = new HashSet<String>();
targetColumns.addAll(targetColNames);
if(targetColNames.size() != targetColumns.size()) {
throw new SemanticException(generateErrorMessage(tabColName,
- "Duplicate column name detected in " + fullTableName + " table schema specification"));
+ "Duplicate column name detected in " + fullTableName + " table schema specification"));
}
Table targetTable = null;
try {
@@ -1667,7 +1670,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
if(targetTable == null) {
throw new SemanticException(generateErrorMessage(ast,
- "Unable to access metadata for table " + fullTableName));
+ "Unable to access metadata for table " + fullTableName));
}
for(FieldSchema f : targetTable.getCols()) {
//parser only allows foo(a,b), not foo(foo.a, foo.b)
@@ -1701,7 +1704,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
if(tokPartVal != null && tokPartVal.getType() == HiveParser.TOK_PARTVAL && tokPartVal.getChildCount() == 1) {
assert tokPartVal.getChild(0).getType() == HiveParser.Identifier :
- "Expected column name; found tokType=" + tokPartVal.getType();
+ "Expected column name; found tokType=" + tokPartVal.getType();
dynamicPartitionColumns.add(tokPartVal.getChild(0).getText());
}
}
@@ -1713,10 +1716,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if(!targetColumns.isEmpty()) {
//Found some columns in user specified schema which are neither regular not dynamic partition columns
throw new SemanticException(generateErrorMessage(tabColName,
- "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) +
- "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") +
- " not found among regular columns of " +
- fullTableName + " nor dynamic partition columns."));
+ "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) +
+ "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") +
+ " not found among regular columns of " +
+ fullTableName + " nor dynamic partition columns."));
}
}
}
@@ -1761,8 +1764,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (cte != null) {
if (ctesExpanded.contains(cteName)) {
throw new SemanticException("Recursive cte " + cteName +
- " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") +
- " -> " + cteName + ").");
+ " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") +
+ " -> " + cteName + ").");
}
cte.reference++;
current.parents.add(cte);
@@ -1804,7 +1807,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
- throws HiveException {
+ throws HiveException {
if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
getMetaData(qbexpr.getQB(), parentInput);
} else {
@@ -1815,7 +1818,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private void getMetaData(QB qb, ReadEntity parentInput)
- throws HiveException {
+ throws HiveException {
LOG.info("Get metadata for source tables");
// Go over the tables and populate the related structures.
@@ -1847,7 +1850,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
tab = new Table(tab.getTTable().deepCopy());
}
if (tab == null ||
- tab.getDbName().equals(SessionState.get().getCurrentDatabase())) {
+ tab.getDbName().equals(SessionState.get().getCurrentDatabase())) {
Table materializedTab = ctx.getMaterializedTable(cteName);
if (materializedTab == null) {
// we first look for this alias from CTE, and then from catalog.
@@ -1877,7 +1880,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- if (tab.isView()) {
+ if (tab.isView()) {
if (qb.getParseInfo().isAnalyzeCommand()) {
throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg());
}
@@ -1901,7 +1904,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
String aliasId = getAliasId(alias, qb);
if (aliasId != null) {
aliasId = aliasId.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "")
- .replace(SemanticAnalyzer.SUBQUERY_TAG_2, "");
+ .replace(SemanticAnalyzer.SUBQUERY_TAG_2, "");
}
viewAliasToInput.put(aliasId, viewInput);
continue;
@@ -1909,8 +1912,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) {
throw new SemanticException(generateErrorMessage(
- qb.getParseInfo().getSrcForAlias(alias),
- ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
+ qb.getParseInfo().getSrcForAlias(alias),
+ ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
}
qb.getMetaData().setSrcForAlias(alias, tab);
@@ -1923,8 +1926,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
} catch (HiveException e) {
throw new SemanticException(generateErrorMessage(
- qb.getParseInfo().getSrcForAlias(alias),
- "Cannot get partitions for " + ts.partSpec), e);
+ qb.getParseInfo().getSrcForAlias(alias),
+ "Cannot get partitions for " + ts.partSpec), e);
}
}
@@ -1936,7 +1939,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Temporary tables created during the execution are not the input sources
if (!PlanUtils.isValuesTempTable(alias)) {
PlanUtils.addInput(inputs,
- new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect);
+ new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect);
}
}
@@ -1972,84 +1975,84 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
for (String name : qbp.getClauseNamesForDest()) {
ASTNode ast = qbp.getDestForClause(name);
switch (ast.getToken().getType()) {
- case HiveParser.TOK_TAB: {
- TableSpec ts = new TableSpec(db, conf, ast);
- if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) {
- throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
- }
+ case HiveParser.TOK_TAB: {
+ TableSpec ts = new TableSpec(db, conf, ast);
+ if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) {
+ throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
+ }
- Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass();
- if (!ts.tableHandle.isNonNative() &&
- !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) {
- throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
- .getMsg(ast, "The class is " + outputFormatClass.toString()));
- }
+ Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass();
+ if (!ts.tableHandle.isNonNative() &&
+ !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) {
+ throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
+ .getMsg(ast, "The class is " + outputFormatClass.toString()));
+ }
- boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(ts.tableHandle.getDbName(),
+ boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(ts.tableHandle.getDbName(),
ts.tableHandle.getTableName());
- isTableWrittenTo |= (qb.getParseInfo().getInsertOverwriteTables().
+ isTableWrittenTo |= (qb.getParseInfo().getInsertOverwriteTables().
get(getUnescapedName((ASTNode) ast.getChild(0), ts.tableHandle.getDbName()).toLowerCase()) != null);
- assert isTableWrittenTo :
+ assert isTableWrittenTo :
"Inconsistent data structure detected: we are writing to " + ts.tableHandle + " in " +
- name + " but it's not in isInsertIntoTable() or getInsertOverwriteTables()";
- // Disallow update and delete on non-acid tables
- boolean isAcid = AcidUtils.isAcidTable(ts.tableHandle);
- if ((updating(name) || deleting(name)) && !isAcid) {
- // Whether we are using an acid compliant transaction manager has already been caught in
- // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
- // here, it means the table itself doesn't support it.
- throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName);
+ name + " but it's not in isInsertIntoTable() or getInsertOverwriteTables()";
+ // Disallow update and delete on non-acid tables
+ boolean isAcid = AcidUtils.isAcidTable(ts.tableHandle);
+ if ((updating(name) || deleting(name)) && !isAcid) {
+ // Whether we are using an acid compliant transaction manager has already been caught in
+ // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
+ // here, it means the table itself doesn't support it.
+ throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName);
+ }
+ // TableSpec ts is got from the query (user specified),
+ // which means the user didn't specify partitions in their query,
+ // but whether the table itself is partitioned is not know.
+ if (ts.specType != SpecType.STATIC_PARTITION) {
+ // This is a table or dynamic partition
+ qb.getMetaData().setDestForAlias(name, ts.tableHandle);
+ // has dynamic as well as static partitions
+ if (ts.partSpec != null && ts.partSpec.size() > 0) {
+ qb.getMetaData().setPartSpecForAlias(name, ts.partSpec);
}
- // TableSpec ts is got from the query (user specified),
- // which means the user didn't specify partitions in their query,
- // but whether the table itself is partitioned is not know.
- if (ts.specType != SpecType.STATIC_PARTITION) {
- // This is a table or dynamic partition
- qb.getMetaData().setDestForAlias(name, ts.tableHandle);
- // has dynamic as well as static partitions
- if (ts.partSpec != null && ts.partSpec.size() > 0) {
- qb.getMetaData().setPartSpecForAlias(name, ts.partSpec);
- }
- } else {
- // This is a partition
- qb.getMetaData().setDestForAlias(name, ts.partHandle);
- }
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- // Add the table spec for the destination table.
- qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
- }
- break;
+ } else {
+ // This is a partition
+ qb.getMetaData().setDestForAlias(name, ts.partHandle);
+ }
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+ // Add the table spec for the destination table.
+ qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
}
+ break;
+ }
- case HiveParser.TOK_DIR: {
- // This is a dfs file
- String fname = stripQuotes(ast.getChild(0).getText());
- if ((!qb.getParseInfo().getIsSubQ())
- && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
+ case HiveParser.TOK_DIR: {
+ // This is a dfs file
+ String fname = stripQuotes(ast.getChild(0).getText());
+ if ((!qb.getParseInfo().getIsSubQ())
+ && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
- if (qb.isCTAS() || qb.isMaterializedView()) {
- qb.setIsQuery(false);
- ctx.setResDir(null);
- ctx.setResFile(null);
+ if (qb.isCTAS() || qb.isMaterializedView()) {
+ qb.setIsQuery(false);
+ ctx.setResDir(null);
+ ctx.setResFile(null);
- Path location;
- // If the CTAS query does specify a location, use the table location, else use the db location
- if (qb.getTableDesc() != null && qb.getTableDesc().getLocation() != null) {
- location = new Path(qb.getTableDesc().getLocation());
- } else {
- // allocate a temporary output dir on the location of the table
- String tableName = getUnescapedName((ASTNode) ast.getChild(0));
- String[] names = Utilities.getDbTableName(tableName);
- try {
- Warehouse wh = new Warehouse(conf);
- //Use destination table's db location.
- String destTableDb = qb.getTableDesc() != null ? qb.getTableDesc().getDatabaseName() : null;
- if (destTableDb == null) {
- destTableDb = names[0];
- }
- location = wh.getDatabasePath(db.getDatabase(destTableDb));
- } catch (MetaException e) {
- throw new SemanticException(e);
+ Path location;
+ // If the CTAS query does specify a location, use the table location, else use the db location
+ if (qb.getTableDesc() != null && qb.getTableDesc().getLocation() != null) {
+ location = new Path(qb.getTableDesc().getLocation());
+ } else {
+ // allocate a temporary output dir on the location of the table
+ String tableName = getUnescapedName((ASTNode) ast.getChild(0));
+ String[] names = Utilities.getDbTableName(tableName);
+ try {
+ Warehouse wh = new Warehouse(conf);
+ //Use destination table's db location.
+ String destTableDb = qb.getTableDesc() != null ? qb.getTableDesc().getDatabaseName() : null;
+ if (destTableDb == null) {
+ destTableDb = names[0];
+ }
+ location = wh.getDatabasePath(db.getDatabase(destTableDb));
+ } catch (MetaException e) {
+ throw new SemanticException(e);
}
}
try {
@@ -2071,66 +2074,66 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Add the table spec for the destination table.
qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
}
- } else {
- // This is the only place where isQuery is set to true; it defaults to false.
- qb.setIsQuery(true);
- Path stagingPath = getStagingDirectoryPathname(qb);
- fname = stagingPath.toString();
- ctx.setResDir(stagingPath);
- }
+ } else {
+ // This is the only place where isQuery is set to true; it defaults to false.
+ qb.setIsQuery(true);
+ Path stagingPath = getStagingDirectoryPathname(qb);
+ fname = stagingPath.toString();
+ ctx.setResDir(stagingPath);
}
+ }
- boolean isDfsFile = true;
- if (ast.getChildCount() >= 2 && ast.getChild(1).getText().toLowerCase().equals("local")) {
- isDfsFile = false;
- }
- // Set the destination for the SELECT query inside the CTAS
- qb.getMetaData().setDestForAlias(name, fname, isDfsFile);
-
- CreateTableDesc directoryDesc = new CreateTableDesc();
- boolean directoryDescIsSet = false;
- int numCh = ast.getChildCount();
- for (int num = 1; num < numCh ; num++){
- ASTNode child = (ASTNode) ast.getChild(num);
- if (child != null) {
- if (storageFormat.fillStorageFormat(child)) {
- directoryDesc.setOutputFormat(storageFormat.getOutputFormat());
- directoryDesc.setSerName(storageFormat.getSerde());
- directoryDescIsSet = true;
- continue;
- }
- switch (child.getToken().getType()) {
- case HiveParser.TOK_TABLEROWFORMAT:
- rowFormatParams.analyzeRowFormat(child);
- directoryDesc.setFieldDelim(rowFormatParams.fieldDelim);
- directoryDesc.setLineDelim(rowFormatParams.lineDelim);
- directoryDesc.setCollItemDelim(rowFormatParams.collItemDelim);
- directoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim);
- directoryDesc.setFieldEscape(rowFormatParams.fieldEscape);
- directoryDesc.setNullFormat(rowFormatParams.nullFormat);
- directoryDescIsSet=true;
- break;
- case HiveParser.TOK_TABLESERIALIZER:
- ASTNode serdeChild = (ASTNode) child.getChild(0);
- storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText()));
- directoryDesc.setSerName(storageFormat.getSerde());
- if (serdeChild.getChildCount() > 1) {
- directoryDesc.setSerdeProps(new HashMap<String, String>());
- readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps());
- }
- directoryDescIsSet = true;
- break;
+ boolean isDfsFile = true;
+ if (ast.getChildCount() >= 2 && ast.getChild(1).getText().toLowerCase().equals("local")) {
+ isDfsFile = false;
+ }
+ // Set the destination for the SELECT query inside the CTAS
+ qb.getMetaData().setDestForAlias(name, fname, isDfsFile);
+
+ CreateTableDesc directoryDesc = new CreateTableDesc();
+ boolean directoryDescIsSet = false;
+ int numCh = ast.getChildCount();
+ for (int num = 1; num < numCh ; num++){
+ ASTNode child = (ASTNode) ast.getChild(num);
+ if (child != null) {
+ if (storageFormat.fillStorageFormat(child)) {
+ directoryDesc.setOutputFormat(storageFormat.getOutputFormat());
+ directoryDesc.setSerName(storageFormat.getSerde());
+ directoryDescIsSet = true;
+ continue;
+ }
+ switch (child.getToken().getType()) {
+ case HiveParser.TOK_TABLEROWFORMAT:
+ rowFormatParams.analyzeRowFormat(child);
+ directoryDesc.setFieldDelim(rowFormatParams.fieldDelim);
+ directoryDesc.setLineDelim(rowFormatParams.lineDelim);
+ directoryDesc.setCollItemDelim(rowFormatParams.collItemDelim);
+ directoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim);
+ directoryDesc.setFieldEscape(rowFormatParams.fieldEscape);
+ directoryDesc.setNullFormat(rowFormatParams.nullFormat);
+ directoryDescIsSet=true;
+ break;
+ case HiveParser.TOK_TABLESERIALIZER:
+ ASTNode serdeChild = (ASTNode) child.getChild(0);
+ storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText()));
+ directoryDesc.setSerName(storageFormat.getSerde());
+ if (serdeChild.getChildCount() > 1) {
+ directoryDesc.setSerdeProps(new HashMap<String, String>());
+ readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps());
}
+ directoryDescIsSet = true;
+ break;
}
}
- if (directoryDescIsSet){
- qb.setDirectoryDesc(directoryDesc);
- }
- break;
}
- default:
- throw new SemanticException(generateErrorMessage(ast,
- "Unknown Token Type " + ast.getToken().getType()));
+ if (directoryDescIsSet){
+ qb.setDirectoryDesc(directoryDesc);
+ }
+ break;
+ }
+ default:
+ throw new SemanticException(generateErrorMessage(ast,
+ "Unknown Token Type " + ast.getToken().getType()));
}
}
}
@@ -2151,7 +2154,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
} catch (Exception e) {
- throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e);
+ throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e);
}
return false;
@@ -2283,12 +2286,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void replaceViewReferenceWithDefinition(QB qb, Table tab,
- String tab_name, String alias) throws SemanticException {
+ String tab_name, String alias) throws SemanticException {
ASTNode viewTree;
final ASTNodeOrigin viewOrigin = new ASTNodeOrigin("VIEW", tab.getTableName(),
tab.getViewExpandedText(), alias, qb.getParseInfo().getSrcForAlias(
- alias));
+ alias));
try {
// Reparse text, passing null for context to avoid clobbering
// the top-level token stream.
@@ -2305,7 +2308,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Dispatcher nodeOriginDispatcher = new Dispatcher() {
@Override
public Object dispatch(Node nd, java.util.Stack<Node> stack,
- Object... nodeOutputs) {
+ Object... nodeOutputs) {
((ASTNode) nd).setOrigin(viewOrigin);
return null;
}
@@ -2363,7 +2366,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("rawtypes")
private String findAlias(ASTNode columnRef,
- Map<String, Operator> aliasToOpInfo) throws SemanticException {
+ Map<String, Operator> aliasToOpInfo) throws SemanticException {
String colName = unescapeIdentifier(columnRef.getChild(0).getText()
.toLowerCase());
String tabAlias = null;
@@ -2391,9 +2394,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
void parseJoinCondPopulateAlias(QBJoinTree joinTree, ASTNode condn,
- ArrayList<String> leftAliases, ArrayList<String> rightAliases,
- ArrayList<String> fields,
- Map<String, Operator> aliasToOpInfo) throws SemanticException {
+ ArrayList<String> leftAliases, ArrayList<String> rightAliases,
+ ArrayList<String> fields,
+ Map<String, Operator> aliasToOpInfo) throws SemanticException {
// String[] allAliases = joinTree.getAllAliases();
switch (condn.getToken().getType()) {
case HiveParser.TOK_TABLE_OR_COL:
@@ -2497,8 +2500,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void populateAliases(List<String> leftAliases,
- List<String> rightAliases, ASTNode condn, QBJoinTree joinTree,
- List<String> leftSrc) throws SemanticException {
+ List<String> rightAliases, ASTNode condn, QBJoinTree joinTree,
+ List<String> leftSrc) throws SemanticException {
if ((leftAliases.size() != 0) && (rightAliases.size() != 0)) {
joinTree.addPostJoinFilter(condn);
return;
@@ -2525,21 +2528,21 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* only left sources are referenced in a Predicate
*/
void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree,
- JoinType type,
- List<String> leftSrc,
- ASTNode joinCond,
- ASTNode leftCondn,
- ASTNode rightCondn,
- List<String> leftCondAl1,
- List<String> leftCondAl2,
- List<String> rightCondAl1,
- List<String> rightCondAl2) throws SemanticException {
+ JoinType type,
+ List<String> leftSrc,
+ ASTNode joinCond,
+ ASTNode leftCondn,
+ ASTNode rightCondn,
+ List<String> leftCondAl1,
+ List<String> leftCondAl2,
+ List<String> rightCondAl1,
+ List<String> rightCondAl2) throws SemanticException {
if (leftCondAl1.size() != 0) {
if ((rightCondAl1.size() != 0)
|| ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
if (type.equals(JoinType.LEFTOUTER) ||
type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(0).add(joinCond);
+ joinTree.getFilters().get(0).add(joinCond);
} else {
/*
* If the rhs references table sources and this QBJoinTree has a leftTree;
@@ -2623,7 +2626,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
|| ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
if (type.equals(JoinType.RIGHTOUTER)
|| type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(1).add(joinCond);
+ joinTree.getFilters().get(1).add(joinCond);
} else {
joinTree.getFiltersForPushing().get(1).add(joinCond);
}
@@ -2638,14 +2641,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
} else if (rightCondAl1.size() != 0) {
if (type.equals(JoinType.LEFTOUTER)
|| type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(0).add(joinCond);
+ joinTree.getFilters().get(0).add(joinCond);
} else {
joinTree.getFiltersForPushing().get(0).add(joinCond);
}
} else {
if (type.equals(JoinType.RIGHTOUTER)
|| type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(1).add(joinCond);
+ joinTree.getFilters().get(1).add(joinCond);
} else if (type.equals(JoinType.LEFTSEMI)) {
joinTree.getExpressions().get(0).add(leftCondn);
joinTree.getExpressions().get(1).add(rightCondn);
@@ -2661,7 +2664,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("rawtypes")
private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, List<String> leftSrc,
- Map<String, Operator> aliasToOpInfo)
+ Map<String, Operator> aliasToOpInfo)
throws SemanticException {
if (joinCond == null) {
return;
@@ -2700,8 +2703,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("rawtypes")
private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond,
- List<String> leftSrc, JoinType type,
- Map<String, Operator> aliasToOpInfo) throws SemanticException {
+ List<String> leftSrc, JoinType type,
+ Map<String, Operator> aliasToOpInfo) throws SemanticException {
if (joinCond == null) {
return;
}
@@ -2797,14 +2800,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (!leftAliasNull) {
if (type.equals(JoinType.LEFTOUTER)
|| type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(0).add(joinCond);
+ joinTree.getFilters().get(0).add(joinCond);
} else {
joinTree.getFiltersForPushing().get(0).add(joinCond);
}
} else {
if (type.equals(JoinType.RIGHTOUTER)
|| type.equals(JoinType.FULLOUTER)) {
- joinTree.getFilters().get(1).add(joinCond);
+ joinTree.getFilters().get(1).add(joinCond);
} else {
joinTree.getFiltersForPushing().get(1).add(joinCond);
}
@@ -2817,7 +2820,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("rawtypes")
private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String dest, ASTNode predicate,
- Map<String, Operator> aliasToOpInfo) throws SemanticException {
+ Map<String, Operator> aliasToOpInfo) throws SemanticException {
switch (predicate.getType()) {
case HiveParser.KW_AND:
@@ -2834,7 +2837,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
ArrayList<String> leftCondAl2 = new ArrayList<String>();
try {
parseJoinCondPopulateAlias(joinTree, leftCondn, leftCondAl1, leftCondAl2,
- null, aliasToOpInfo);
+ null, aliasToOpInfo);
} catch(SemanticException se) {
// suppress here; if it is a real issue will get caught in where clause handling.
return;
@@ -2884,7 +2887,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
public <T extends OperatorDesc> Operator<T> putOpInsertMap(Operator<T> op,
- RowResolver rr) {
+ RowResolver rr) {
OpParseContext ctx = new OpParseContext(rr);
opParseCtx.put(op, ctx);
op.augmentPlan();
@@ -2893,7 +2896,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private Operator genHavingPlan(String dest, QB qb, Operator input,
- Map<String, Operator> aliasToOpInfo)
+ Map<String, Operator> aliasToOpInfo)
throws SemanticException {
ASTNode havingExpr = qb.getParseInfo().getHavingForClause(dest);
@@ -2912,8 +2915,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// If CBO did not optimize the query, we might need to replace grouping function
final String destClauseName = qb.getParseInfo().getClauseNames().iterator().next();
final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty()
- || !qb.getParseInfo().getDestGroupingSets().isEmpty()
- || !qb.getParseInfo().getDestCubes().isEmpty());
+ || !qb.getParseInfo().getDestGroupingSets().isEmpty()
+ || !qb.getParseInfo().getDestCubes().isEmpty());
// Special handling of grouping function
condn = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), destClauseName), condn,
!cubeRollupGrpSetPresent);
@@ -2930,7 +2933,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
protected static ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode,
- final boolean noneSet) throws SemanticException {
+ final boolean noneSet) throws SemanticException {
TreeVisitorAction action = new TreeVisitorAction() {
@@ -2953,9 +2956,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
ASTNode newRoot = new ASTNode();
// Rewritten grouping function
ASTNode groupingFunc = (ASTNode) ParseDriver.adaptor.create(
- HiveParser.Identifier, "grouping");
+ HiveParser.Identifier, "grouping");
ParseDriver.adaptor.addChild(groupingFunc, ParseDriver.adaptor.create(
- HiveParser.Identifier, "rewritten"));
+ HiveParser.Identifier, "rewritten"));
newRoot.addChild(groupingFunc);
// Grouping ID reference
ASTNode childGroupingID;
@@ -2963,13 +2966,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus,
// grouping should return 0
childGroupingID = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral,
- String.valueOf(0));
+ String.valueOf(0));
} else {
// We refer to grouping_id column
childGroupingID = (ASTNode) ParseDriver.adaptor.create(
- HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
+ HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
ParseDriver.adaptor.addChild(childGroupingID, ParseDriver.adaptor.create(
- HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
+ HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
}
newRoot.addChild(childGroupingID);
// Indices
@@ -2981,7 +2984,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Create and add AST node with position of grouping function input
// in group by clause
ASTNode childN = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral,
- String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size())));
+ String.valueOf(IntMath.mod(-j-1, grpByAstExprs.size())));
newRoot.addChild(childN);
break;
}
@@ -3013,8 +3016,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input,
- Map<String, Operator> aliasToOpInfo,
- boolean forHavingClause, boolean forGroupByClause)
+ Map<String, Operator> aliasToOpInfo,
+ boolean forHavingClause, boolean forGroupByClause)
throws SemanticException {
OpParseContext inputCtx = opParseCtx.get(input);
@@ -3114,7 +3117,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if ( subQuery.getOperator().getType() != SubQueryType.EXISTS &&
subQuery.getOperator().getType() != SubQueryType.NOT_EXISTS &&
sqRR.getColumnInfos().size() -
- subQuery.getNumOfCorrelationExprsAddedToSQSelect() > 1 ) {
+ subQuery.getNumOfCorrelationExprsAddedToSQSelect() > 1 ) {
throw new SemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
subQueryAST, "SubQuery can contain only 1 item in Select List."));
}
@@ -3205,7 +3208,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* every non nullSafe predicate.
*/
private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input,
- QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException {
+ QBJoinTree joinTree, ExprNodeDesc[] joinKeys) throws SemanticException {
if (qb == null || joinTree == null) {
return input;
@@ -3226,7 +3229,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
List<Boolean> nullSafes = joinTree.getNullSafes();
for (int i = 0; i < joinKeys.length; i++) {
if (nullSafes.get(i) || (joinKeys[i] instanceof ExprNodeColumnDesc &&
- ((ExprNodeColumnDesc)joinKeys[i]).getIsPartitionColOrVirtualCol())) {
+ ((ExprNodeColumnDesc)joinKeys[i]).getIsPartitionColOrVirtualCol())) {
// no need to generate is not null predicate for partitioning or
// virtual column, since those columns can never be null.
continue;
@@ -3273,11 +3276,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
@SuppressWarnings("nls")
- // TODO: make aliases unique, otherwise needless rewriting takes place
+ // TODO: make aliases unique, otherwise needless rewriting takes place
Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel,
- ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input,
- RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases,
- boolean ensureUniqueCols) throws SemanticException {
+ ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input,
+ RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases,
+ boolean ensureUniqueCols) throws SemanticException {
if (colSrcRR == null) {
colSrcRR = input;
@@ -3514,7 +3517,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private TableDesc getTableDescFromSerDe(ASTNode child, String cols,
- String colTypes, boolean defaultCols) throws SemanticException {
+ String colTypes, boolean defaultCols) throws SemanticException {
if (child.getType() == HiveParser.TOK_SERDENAME) {
String serdeName = unescapeSQLString(child.getChild(0).getText());
Class<? extends Deserializer> serdeClass = null;
@@ -3723,7 +3726,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Input and Output Serdes
if (trfm.getChild(inputSerDeNum).getChildCount() > 0) {
inInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm
- .getChild(inputSerDeNum))).getChild(0), inpColumns.toString(),
+ .getChild(inputSerDeNum))).getChild(0), inpColumns.toString(),
inpColumnTypes.toString(), false);
} else {
inInfo = PlanUtils.getTableDesc(serde, Integer
@@ -3733,7 +3736,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (trfm.getChild(outputSerDeNum).getChildCount() > 0) {
outInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm
- .getChild(outputSerDeNum))).getChild(0), columns.toString(),
+ .getChild(outputSerDeNum))).getChild(0), columns.toString(),
columnTypes.toString(), false);
// This is for backward compatibility. If the user did not specify the
// output column list, we assume that there are 2 columns: key and value.
@@ -3857,7 +3860,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
protected List<Integer> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo,
- String dest) throws SemanticException {
+ String dest) throws SemanticException {
Map<String, Integer> exprPos = new HashMap<String, Integer>();
for (int i = 0; i < groupByExpr.size(); ++i) {
ASTNode node = groupByExpr.get(i);
@@ -3947,7 +3950,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
static String[] getColAlias(ASTNode selExpr, String defaultName,
- RowResolver inputRR, boolean includeFuncName, int colNum) {
+ RowResolver inputRR, boolean includeFuncName, int colNum) {
String colAlias = null;
String tabAlias = null;
String[] colRef = new String[2];
@@ -3955,7 +3958,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
//for queries with a windowing expressions, the selexpr may have a third child
if (selExpr.getChildCount() == 2 ||
(selExpr.getChildCount() == 3 &&
- selExpr.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC)) {
+ selExpr.getChild(2).getType() == HiveParser.TOK_WINDOWSPEC)) {
// return zz for "xx + yy AS zz"
colAlias = unescapeIdentifier(selExpr.getChild(1).getText().toLowerCase());
colRef[0] = tabAlias;
@@ -4038,7 +4041,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input,
- Operator<?> inputForSelectStar) throws SemanticException {
+ Operator<?> inputForSelectStar) throws SemanticException {
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
Operator<?> op = genSelectPlan(dest, selExprList, qb, input, inputForSelectStar, false);
@@ -4051,7 +4054,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private Operator<?> genSelectPlan(String dest, ASTNode selExprList, QB qb, Operator<?> input,
- Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException {
+ Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException {
if (LOG.isDebugEnabled()) {
LOG.debug("tree: " + selExprList.toStringTree());
@@ -4174,8 +4177,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty()
- || !qb.getParseInfo().getDestGroupingSets().isEmpty()
- || !qb.getParseInfo().getDestCubes().isEmpty());
+ || !qb.getParseInfo().getDestGroupingSets().isEmpty()
+ || !qb.getParseInfo().getDestCubes().isEmpty());
Set<String> colAliases = new HashSet<String>();
ASTNode[] exprs = new ASTNode[exprList.getChildCount()];
String[][] aliases = new String[exprList.getChildCount()][];
@@ -4229,7 +4232,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
int initPos = pos;
pos = genColListRegex(".*", expr.getChildCount() == 0 ? null
- : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(),
+ : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(),
expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
if (unparseTranslator.isEnabled()) {
offset += pos - initPos - 1;
@@ -4246,7 +4249,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
} else if (expr.getType() == HiveParser.DOT
&& expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
&& inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0)
- .getChild(0).getText().toLowerCase())) && !hasAsClause
+ .getChild(0).getText().toLowerCase())) && !hasAsClause
&& !inputRR.getIsExprResolver()
&& isRegex(unescapeIdentifier(expr.getChild(1).getText()), conf)) {
// In case the expression is TABLE.COL (col can be regex).
@@ -4254,7 +4257,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// We don't allow this for ExprResolver - the Group By case
pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()),
unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()),
- expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
+ expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
} else {
// Case when this is an expression
TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR, true, isCBOExecuted());
@@ -4341,8 +4344,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @throws SemanticException
*/
public RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
- RowResolver outputRR, RowResolver inputRR, QB qb,
- ASTNode selExprList) throws SemanticException {
+ RowResolver outputRR, RowResolver inputRR, QB qb,
+ ASTNode selExprList) throws SemanticException {
//(z,x)
List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query
if(targetTableSchema == null) {
@@ -4353,9 +4356,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Table target = qb.getMetaData().getDestTableForAlias(dest);
Partition partition = target == null ? qb.getMetaData().getDestPartitionForAlias(dest) : null;
throw new SemanticException(generateErrorMessage(selExprList,
- "Expected " + targetTableSchema.size() + " columns for " + dest +
- (target != null ? "/" + target.getCompleteName() : (partition != null ? "/" + partition.getCompleteName() : "")) +
- "; select produces " + col_list.size() + " columns"));
+ "Expected " + targetTableSchema.size() + " columns for " + dest +
+ (target != null ? "/" + target.getCompleteName() : (partition != null ? "/" + partition.getCompleteName() : "")) +
+ "; select produces " + col_list.size() + " columns"));
}
//e.g. map z->expr for a
Map<String, ExprNodeDesc> targetCol2Projection = new HashMap<String, ExprNodeDesc>();
@@ -4370,7 +4373,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Partition partition = target == null ? qb.getMetaData().getDestPartitionForAlias(dest) : null;
if(target == null && partition == null) {
throw new SemanticException(generateErrorMessage(selExprList,
- "No table/partition found in QB metadata for dest='" + dest + "'"));
+ "No table/partition found in QB metadata for dest='" + dest + "'"));
}
ArrayList<ExprNodeDesc> new_col_list = new ArrayList<ExprNodeDesc>();
colListPos = 0;
@@ -4410,7 +4413,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
new_col_list.add(exp);
final String tableAlias = null;//this column doesn't come from any table
ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos),
- exp.getWritableObjectInspector(), tableAlias, false);
+ exp.getWritableObjectInspector(), tableAlias, false);
newOutputRR.put(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo);
}
colListPos++;
@@ -4486,8 +4489,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* for each GroupBy aggregation.
*/
public static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName,
- ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree,
- boolean isDistinct, boolean isAllColumns)
+ ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree,
+ boolean isDistinct, boolean isAllColumns)
throws SemanticException {
ArrayList<ObjectInspector> originalParameterTypeInfos =
getWritableObjectInspector(aggParameters);
@@ -4514,7 +4517,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* when the UDAF is not found or has problems.
*/
public static GenericUDAFInfo getGenericUDAFInfo(GenericUDAFEvaluator evaluator,
- GenericUDAFEvaluator.Mode emode, ArrayList<ExprNodeDesc> aggParameters)
+ GenericUDAFEvaluator.Mode emode, ArrayList<ExprNodeDesc> aggParameters)
throws SemanticException {
GenericUDAFInfo r = new GenericUDAFInfo();
@@ -4586,7 +4589,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* a constant parameter; otherwise, return null
*/
public static ExprNodeDesc isConstantParameterInAggregationParameters(String internalName,
- List<ExprNodeDesc> reduceValues) {
+ List<ExprNodeDesc> reduceValues) {
// only the pattern of "VALUE._col([0-9]+)" should be handled.
String[] terms = internalName.split("\\.");
@@ -4624,8 +4627,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanGroupByOperator(QBParseInfo parseInfo,
- String dest, Operator input, ReduceSinkOperator rs, GroupByDesc.Mode mode,
- Map<String, GenericUDAFEvaluator> genericUDAFEvaluators)
+ String dest, Operator input, ReduceSinkOperator rs, GroupByDesc.Mode mode,
+ Map<String, GenericUDAFEvaluator> genericUDAFEvaluators)
throws SemanticException {
RowResolver groupByInputRowResolver = opParseCtx
.get(input).getRowResolver();
@@ -4758,10 +4761,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// This function is called for GroupBy2 to pass the additional grouping keys introduced by
// GroupBy1 for the grouping set (corresponding to the rollup).
private void addGroupingSetKey(List<ExprNodeDesc> groupByKeys,
- RowResolver groupByInputRowResolver,
- RowResolver groupByOutputRowResolver,
- List<String> outputColumnNames,
- Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
+ RowResolver groupByInputRowResolver,
+ RowResolver groupByOutputRowResolver,
+ List<String> outputColumnNames,
+ Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
// For grouping sets, add a dummy grouping key
String groupingSetColumnName =
groupByInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
@@ -4789,10 +4792,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// This function is called for ReduceSink to add the additional grouping keys introduced by
// GroupBy1 into the reduce keys.
private void processGroupingSetReduceSinkOperator(RowResolver reduceSinkInputRowResolver,
- RowResolver reduceSinkOutputRowResolver,
- List<ExprNodeDesc> reduceKeys,
- List<String> outputKeyColumnNames,
- Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
+ RowResolver reduceSinkOutputRowResolver,
+ List<ExprNodeDesc> reduceKeys,
+ List<String> outputKeyColumnNames,
+ Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
// add a key for reduce sink
String groupingSetColumnName =
reduceSinkInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
@@ -4832,11 +4835,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanGroupByOperator1(QBParseInfo parseInfo,
- String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode,
- Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
- List<Integer> groupingSets,
- boolean groupingSetsPresent,
- boolean groupingSetsNeedAdditionalMRJob) throws SemanticException {
+ String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode,
+ Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
+ List<Integer> groupingSets,
+ boolean groupingSetsPresent,
+ boolean groupingSetsNeedAdditionalMRJob) throws SemanticException {
ArrayList<String> outputColumnNames = new ArrayList<String>();
RowResolver groupByInputRowResolver = opParseCtx
.get(reduceSinkOperatorInfo).getRowResolver();
@@ -4974,7 +4977,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
assert (paraExpression != null);
aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(),
paraExpression, paraExprInfo.getTabAlias(), paraExprInfo
- .getIsVirtualCol()));
+ .getIsVirtualCol()));
}
if (isDistinct) {
numDistinctUDFs++;
@@ -5023,9 +5026,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* creates 'n' rows per input row, where 'n' is the number of grouping sets.
*/
private void createNewGroupingKey(List<ExprNodeDesc> groupByKeys,
- List<String> outputColumnNames,
- RowResolver groupByOutputRowResolver,
- Map<String, ExprNodeDesc> colExprMap) {
+ List<String> outputColumnNames,
+ RowResolver groupByOutputRowResolver,
+ Map<String, ExprNodeDesc> colExprMap) {
// The value for the constant does not matter. It is replaced by the grouping set
// value for the actual implementation
ExprNodeConstantDesc constant = new ExprNodeConstantDesc(0);
@@ -5056,13 +5059,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanMapGroupByOperator(QB qb,
- String dest,
- List<ASTNode> grpByExprs,
- Operator inputOperatorInfo,
- GroupByDesc.Mode mode,
- Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
- List<Integer> groupingSetKeys,
- boolean groupingSetsPresent) throws SemanticException {
+ String dest,
+ List<ASTNode> grpByExprs,
+ Operator inputOperatorInfo,
+ GroupByDesc.Mode mode,
+ Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
+ List<Integer> groupingSetKeys,
+ boolean groupingSetsPresent) throws SemanticException {
RowResolver groupByInputRowResolver = opParseCtx.get(inputOperatorInfo)
.getRowResolver();
@@ -5206,14 +5209,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb,
- String dest,
- Operator inputOperatorInfo,
- List<ASTNode> grpByExprs,
- int numPartitionFields,
- boolean changeNumPartitionFields,
- int numReducers,
- boolean mapAggrDone,
- boolean groupingSetsPresent) throws SemanticException {
+ String dest,
+ Operator inputOperatorInfo,
+ List<ASTNode> grpByExprs,
+ int numPartitionFields,
+ boolean changeNumPartitionFields,
+ int numReducers,
+ boolean mapAggrDone,
+ boolean groupingSetsPresent) throws SemanticException {
RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo)
.getRowResolver();
@@ -5295,8 +5298,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs, String dest,
- RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver,
- List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap)
+ RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver,
+ List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap)
throws SemanticException {
ArrayList<ExprNodeDesc> reduceKeys = new ArrayList<ExprNodeDesc>();
@@ -5332,10 +5335,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private List<List<Integer>> getDistinctColIndicesForReduceSink(QBParseInfo parseInfo,
- String dest,
- List<ExprNodeDesc> reduceKeys, RowResolver reduceSinkInputRowResolver,
- RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames,
- Map<String, ExprNodeDesc> colExprMap)
+ String dest,
+ List<ExprNodeDesc> reduceKeys, RowResolver reduceSinkInputRowResolver,
+ RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames,
+ Map<String, ExprNodeDesc> colExprMap)
throws SemanticException {
List<List<Integer>> distinctColIndices = new ArrayList<List<Integer>>();
@@ -5384,9 +5387,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void getReduceValuesForReduceSinkNoMapAgg(QBParseInfo parseInfo, String dest,
- RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver,
- List<String> outputValueColumnNames, ArrayList<ExprNodeDesc> reduceValues,
- Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
+ RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver,
+ List<String> outputValueColumnNames, ArrayList<ExprNodeDesc> reduceValues,
+ Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
HashMap<String, ASTNode> aggregationTrees = parseInfo
.getAggregationExprsForClause(dest);
@@ -5414,7 +5417,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private ReduceSinkOperator genCommonGroupByPlanReduceSinkOperator(QB qb, List<String> dests,
- Operator inputOperatorInfo) throws SemanticException {
+ Operator inputOperatorInfo) throws SemanticException {
RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo)
.getRowResolver();
@@ -5503,7 +5506,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// from mapping if it's already on RS keys.
// Remaining column expressions would be a candidate for an RS value
private void removeMappingForKeys(ASTNode predicate, Map<ASTNode, ExprNodeDesc> mapping,
- List<ExprNodeDesc> keys) {
+ List<ExprNodeDesc> keys) {
ExprNodeDesc expr = mapping.get(predicate);
if (expr != null && ExprNodeDescUtils.indexOf(expr, keys) >= 0) {
removeRecursively(predicate, mapping);
@@ -5540,11 +5543,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanReduceSinkOperator2MR(QBParseInfo parseInfo,
- String dest,
- Operator groupByOperatorInfo,
- int numPartitionFields,
- int numReducers,
- boolean groupingSetsPresent) throws SemanticException {
+ String dest,
+ Operator groupByOperatorInfo,
+ int numPartitionFields,
+ int numReducers,
+ boolean groupingSetsPresent) throws SemanticException {
RowResolver reduceSinkInputRowResolver2 = opParseCtx.get(
groupByOperatorInfo).getRowResolver();
RowResolver reduceSinkOutputRowResolver2 = new RowResolver();
@@ -5628,11 +5631,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo,
- String dest,
- Operator reduceSinkOperatorInfo2,
- GroupByDesc.Mode mode,
- Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
- boolean groupingSetsPresent) throws SemanticException {
+ String dest,
+ Operator reduceSinkOperatorInfo2,
+ GroupByDesc.Mode mode,
+ Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
+ boolean groupingSetsPresent) throws SemanticException {
RowResolver groupByInputRowResolver2 = opParseCtx.get(
reduceSinkOperatorInfo2).getRowResolver();
@@ -5688,7 +5691,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
assert (paraExpression != null);
aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(),
paraExpression, paraExprInfo.getTabAlias(), paraExprInfo
- .getIsVirtualCol()));
+ .getIsVirtualCol()));
String aggName = unescapeIdentifier(value.getChild(0).getText());
@@ -5707,7 +5710,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
udaf.genericUDAFEvaluator,
udaf.convertedParameters,
(mode != GroupByDesc.Mode.FINAL && value.getToken().getType() ==
- HiveParser.TOK_FUNCTIONDI),
+ HiveParser.TOK_FUNCTIONDI),
amode));
String field = getColumnInternalName(groupByKeys.size()
+ aggregations.size() - 1);
@@ -5797,7 +5800,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings({"nls"})
private Operator genGroupByPlan1ReduceMultiGBY(List<String> dests, QB qb, Operator input,
- Map<String, Operator> aliasToOpInfo)
+ Map<String, Operator> aliasToOpInfo)
throws SemanticException {
QBParseInfo parseInfo = qb.getParseInfo();
@@ -5861,7 +5864,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
orFilterDesc.setGenerated(true);
selectInput = putOpInsertMap(OperatorFactory.getAndMakeChild(orFilterDesc, new RowSchema(
- inputRR.getColumnInfos()), input), inputRR);
+ inputRR.getColumnInfos()), input), inputRR);
}
// insert a select operator here used by the ColumnPruner to reduce
@@ -6025,7 +6028,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
static private void extractColumns(Set<String> colNamesExprs,
- ExprNodeDesc exprNode) throws SemanticException {
+ ExprNodeDesc exprNode) throws SemanticException {
if (exprNode instanceof ExprNodeColumnDesc) {
colNamesExprs.add(((ExprNodeColumnDesc) exprNode).getColumn());
return;
@@ -6050,9 +6053,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
void checkExpressionsForGroupingSet(List<ASTNode> grpByExprs,
- List<ASTNode> distinctGrpByExprs,
- Map<String, ASTNode> aggregationTrees,
- RowResolver inputRowResolver) throws SemanticException {
+ List<ASTNode> distinctGrpByExprs,
+ Map<String, ASTNode> aggregationTrees,
+ RowResolver inputRowResolver) throws SemanticException {
Set<String> colNamesGroupByExprs = new HashSet<String>();
Set<String> colNamesGroupByDistinctExprs = new HashSet<String>();
@@ -6149,7 +6152,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb,
- Operator inputOperatorInfo) throws SemanticException {
+ Operator inputOperatorInfo) throws SemanticException {
QBParseInfo parseInfo = qb.getParseInfo();
ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
@@ -6317,7 +6320,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanMapAggr2MR(String dest, QB qb,
- Operator inputOperatorInfo) throws SemanticException {
+ Operator inputOperatorInfo) throws SemanticException {
QBParseInfo parseInfo = qb.getParseInfo();
@@ -6501,7 +6504,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@SuppressWarnings("nls")
private Operator genBucketingSortingDest(String dest, Operator input, QB qb,
- TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException {
+ TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException {
// If the table is bucketed, and bucketing is enforced, do the following:
// If the number of buckets is smaller than the number of maximum reducers,
@@ -6571,7 +6574,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
nullOrder.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? 'a' : 'z');
}
input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), nullOrder.toString(),
- maxReducers, (AcidUtils.isAcidTable(dest_tab) ?
+ maxReducers, (AcidUtils.isAcidTable(dest_tab) ?
getAcidType(table_desc.getOutputFileFormatClass(), dest) : AcidUtils.Operation.NOT_ACID));
reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0));
ctx.setMultiFileSpray(multiFileSpray);
@@ -6582,7 +6585,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void genPartnCols(String dest, Operator input, QB qb,
- TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException {
+ TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException {
boolean enforceBucketing = false;
ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>();
@@ -6627,6 +6630,128 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf));
}
+ private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException{
+ List<Boolean> nullConstraints = new ArrayList<>();
+ final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints(
+ tbl.getDbName(), tbl.getTableName());
+ ImmutableBitSet bitSet = null;
+ if(nnc == null || nnc.getNotNullConstraints().isEmpty()) {
+ return bitSet;
+ }
+ // Build the bitset with not null columns
+ ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
+ for (String nnCol : nnc.getNotNullConstraints().values()) {
+ int nnPos = -1;
+ for (int i = 0; i < tbl.getCols().size(); i++) {
+ if (tbl.getCols().get(i).getName().equals(nnCol)) {
+ nnPos = i;
+ builder.set(nnPos);
+ break;
+ }
+ }
+ }
+ bitSet = builder.build();
+ return bitSet;
+ }
+
+ private boolean mergeCardinalityViolationBranch(final Operator input) {
+ if(input instanceof SelectOperator) {
+ SelectOperator selectOp = (SelectOperator)input;
+ if(selectOp.getConf().getColList().size() == 1) {
+ ExprNodeDesc colExpr = selectOp.getConf().getColList().get(0);
+ if(colExpr instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc)colExpr ;
+ if(func.getGenericUDF() instanceof GenericUDFCardinalityViolation){
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private Operator
+ genIsNotNullConstraint(String dest, QB qb, Operator input)
+ throws SemanticException {
+
+ boolean forceNotNullConstraint = conf.getBoolVar(ConfVars.HIVE_ENFORCE_NOT_NULL_CONSTRAINT);
+ if(!forceNotNullConstraint) {
+ return input;
+ }
+
+ if(deleting(dest)) {
+ // for DELETE statements NOT NULL constraint need not be checked
+ return input;
+ }
+
+ //MERGE statements could have inserted a cardinality violation branch, we need to avoid that
+ if(mergeCardinalityViolationBranch(input)){
+ return input;
+ }
+
+ // if this is an insert into statement we might need to add constraint check
+ Table targetTable = null;
+ Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest);
+ if(dest_type == QBMetaData.DEST_TABLE) {
+ targetTable= qb.getMetaData().getDestTableForAlias(dest);
+
+ }
+ else if(dest_type == QBMetaData.DEST_PARTITION){
+ Partition dest_part = qb.getMetaData().getDestPartitionForAlias(dest);
+ targetTable = dest_part.getTable();
+
+ }
+ else {
+ throw new SemanticException("Generating NOT NULL constraint check: Invalid target type: " + dest);
+ }
+ ImmutableBitSet nullConstraintBitSet = null;
+ try {
+ nullConstraintBitSet = getEnabledNotNullConstraints(targetTable);
+ } catch (Exception e) {
+ if (e instanceof SemanticException) {
+ throw (SemanticException) e;
+ } else {
+ throw (new RuntimeException(e));
+ }
+ }
+ if(nullConstraintBitSet == null) {
+ return input;
+ }
+ List<ColumnInfo> colInfos = input.getSchema().getSignature();
+
+ ExprNodeDesc currUDF = null;
+ int constraintIdx = 0;
+ for(int colExprIdx=0; colExprIdx < colInfos.size(); colExprIdx++) {
+ if(updating(dest) && colExprIdx == 0) {
+ // for updates first column is _rowid
+ continue;
+ }
+ if (nullConstraintBitSet.indexOf(constraintIdx) != -1) {
+ ExprNodeDesc currExpr = TypeCheckProcFactory.toExprNodeDesc(colInfos.get(colExprIdx));
+ ExprNodeDesc isNotNullUDF = TypeCheckProcFactory.DefaultExprProcessor.
+ getFuncExprNodeDesc("isnotnull", currExpr);
+ ExprNodeDesc constraintUDF = TypeCheckProcFactory.DefaultExprProcessor.
+ getFuncExprNodeDesc("enforce_constraint", isNotNullUDF);
+ if (currUDF != null) {
+ currUDF = TypeCheckProcFactory.DefaultExprProcessor.
+ getFuncExprNodeDesc("and", currUDF, constraintUDF);
+ } else {
+ currUDF = constraintUDF;
+ }
+ }
+ constraintIdx++;
+ }
+ if (currUDF != null) {
+ assert (input.getParentOperators().size() == 1);
+ RowResolver inputRR = opParseCtx.get(input).getRowResolver();
+ Operator newConstraintFilter = putOpInsertMap(OperatorFactory.getAndMakeChild(
+ new FilterDesc(currUDF, false), new RowSchema(
+ inputRR.getColumnInfos()), input), inputRR);
+
+ return newConstraintFilter;
+ }
+ return input;
+ }
@SuppressWarnings("nls")
protected Operator genFileSinkPlan(String dest, QB qb, Operator input)
throws SemanticException {
@@ -6712,6 +6837,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// this table_desc does not contain the partitioning columns
table_desc = Utilities.getTableDesc(dest_tab);
+ // Add NOT NULL constraint check
+ input = genIsNotNullConstraint(dest, qb, input);
+
// Add sorting/bucketing if needed
input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx);
@@ -6736,7 +6864,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
} else {
txnId = acidOp == Operation.NOT_ACID ? null :
- SessionState.get().getTxnMgr().getCurrentTxnId();
+ SessionState.get().getTxnMgr().getCurrentTxnId();
}
boolean isReplace = !qb.getParseInfo().isInsertIntoTable(
dest_tab.getDbName(), dest_tab.getTableName());
@@ -6744,8 +6872,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
// deltas and base and leave them up to the cleaner to clean up
LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
- dest_tab.getTableName()) && !destTableIsTransactional)
- ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
+ dest_tab.getTableName()) && !destTableIsTransactional)
+ ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
ltd.setLoadFileType(loadType);
ltd.setLbCtx(lbCtx);
loadTableWork.add(ltd);
@@ -6755,7 +6883,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
setStatsForNonNativeTable(dest_tab);
// true if it is insert overwrite.
boolean overwrite = !qb.getParseInfo().isInsertIntoTable(
- String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName()));
+ String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName()));
createInsertDesc(dest_tab, overwrite);
}
@@ -6791,6 +6919,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
table_desc = Utilities.getTableDesc(dest_tab);
+ // Add NOT NULL constraint check
+ input = genIsNotNullConstraint(dest, qb, input);
+
// Add sorting/bucketing if needed
input = genBucketingSortingDest(dest, input, qb, table_desc, dest_tab, rsCtx);
@@ -6811,20 +6942,20 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
} else {
txnId = (acidOp == Operation.NOT_ACID) ? null :
- SessionState.get().getTxnMgr().getCurrentTxnId();
+ SessionState.get().getTxnMgr().getCurrentTxnId();
}
ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId);
// For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
// deltas and base and leave them up to the cleaner to clean up
LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
- dest_tab.getTableName()) && !destTableIsTransactional) // // Both Full-acid and MM tables are excluded.
- ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
+ dest_tab.getTableName()) && !destTableIsTransactional) // // Both Full-acid and MM tables are excluded.
+ ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
ltd.setLoadFileType(loadType);
ltd.setLbCtx(lbCtx);
loadTableWork.add(ltd);
if (!outputs.add(new WriteEntity(dest_part,
- determineWriteType(ltd, dest_tab.isNonNative(), dest)))) {
+ determineWriteType(ltd, dest_tab.isNonNative(), dest)))) {
throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES
.getMsg(dest_tab.getTableName() + "@" + dest_part.getName()));
@@ -6872,7 +7003,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
queryTmpdir = isMmTable ? qPath : ctx.getTempDirForFinalJobPath(qPath);
if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
Utilities.FILE_OP_LOGGER.trace("Setting query directory " + queryTmpdir
- + " from " + dest_path + " (" + isMmTable + ")");
+ + " from " + dest_path + " (" + isMmTable + ")");
}
} catch (Exception e) {
throw new SemanticException("Error creating temporary folder on: "
@@ -6905,8 +7036,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Create LFD even for MM CTAS - it's a no-op move, but it still seems to be used for stats.
loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols,
colTypes,
- destTableIsFullAcid ?//there is a change here - prev version had 'transadtional', one beofre' acid'
- Operation.INSERT : Operation.NOT_ACID,
+ destTableIsFullAcid ?//there is a change here - prev version had 'transadtional', one beofre' acid'
+ Operation.INSERT : Operation.NOT_ACID,
isMmCtas));
if (tblDesc == null) {
if (viewDesc != null) {
@@ -6914,23 +7045,23 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
} else if (qb.getIsQuery()) {
String fileFormat;
if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()) {
- fileFormat = "SequenceFile";
- HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat);
- table_desc=
- PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat,
- ThriftJDBCBinarySerDe.class);
- // Set the fetch formatter to be a no-op for the ListSinkOperator, since we'll
- // write out formatted thrift objects to SequenceFile
- conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, NoOpFetchFormatter.class.getName());
+ fileFormat = "SequenceFile";
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT, fileFormat);
+ table_desc=
+ PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat,
+ ThriftJDBCBinarySerDe.class);
+ // Set the fetch formatter to be a no-op for the ListSinkOperator, since we'll
+ // write out formatted thrift objects to SequenceFile
+ conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, NoOpFetchFormatter.class.getName());
} else {
- fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
- Class<? extends Deserializer> serdeClass = LazySimpleSerDe.class;
- if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) {
- serdeClass = LazyBinarySerDe2.class;
- }
- table_desc =
- PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat,
- serdeClass);
+ fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
+ Class<? extends Deserializer> serdeClass = LazySimpleSerDe.class;
+ if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) {
+ serdeClass = LazyBinarySerDe2.class;
+ }
+ table_desc =
+ PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat,
+ serdeClass);
}
} else {
table_desc = PlanUtils.getDefaultTableDesc(qb.getDirectoryDesc(), cols, colTypes);
@@ -7009,12 +7140,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
if (SessionState.get().isHiveServerQuery() &&
- null != table_desc &&
- table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
- HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
- fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true);
+ null != table_desc &&
+ table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
+ HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true);
} else {
- fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false);
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false);
}
Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
@@ -7072,7 +7203,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
String typeName = colInfo.getType().getTypeName();
// CTAS should NOT create a VOID type
if (typeName.equals(serdeConstants.VOID_TYPE_NAME)) {
- throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName));
+ throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName));
}
col.setType(typeName);
field_schemas.add(col);
@@ -7108,31 +7239,31 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc,
- Partition dest_part, Path dest_path, int currentTableId,
- boolean destTableIsAcid, boolean destTableIsTemporary,
- boolean destTableIsMaterialization, Path queryTmpdir,
- SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx,
- RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas,
- Integer dest_type, QB qb) throws SemanticException {
+ Partition dest_part, Path dest_path, int currentTableId,
+ boolean destTableIsAcid, boolean destTableIsTemporary,
+ boolean destTableIsMaterialization, Path queryTmpdir,
+ SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx,
+ RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas,
+ Integer dest_type,
<TRUNCATED>