You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/06/07 18:19:52 UTC
[hive] branch master updated (dc53d04 -> 51b1c6b)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from dc53d04 HIVE-21816: HMS Translation: Refactor tests to work with ACID tables (Naveen Gangam, reviewed by Daniel Dai)
new de9a678 HIVE-21818: CBO: Copying TableRelOptHiveTable has metastore traffic (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
new 51b1c6b HIVE-21834: Avoid unnecessary calls to simplify filter conditions (Jesus Camacho Rodriguez, reviewed by Gopal V)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ql/metadata/HiveMaterializedViewsRegistry.java | 12 +-
.../org/apache/hadoop/hive/ql/metadata/Table.java | 40 ++++
.../hive/ql/optimizer/calcite/HiveRelBuilder.java | 4 +-
.../hive/ql/optimizer/calcite/RelOptHiveTable.java | 206 +++++++++++----------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 88 +++++----
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 57 +++---
.../llap/external_jdbc_table_perf.q.out | 6 +-
7 files changed, 243 insertions(+), 170 deletions(-)
[hive] 01/02: HIVE-21818: CBO: Copying TableRelOptHiveTable has
metastore traffic (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit de9a6783a329e29bfd94896572ee44615349bf28
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Mon Jun 3 18:54:21 2019 -0700
HIVE-21818: CBO: Copying TableRelOptHiveTable has metastore traffic (Jesus Camacho Rodriguez, reviewed by Vineet Garg)
Close apache/hive#663
---
.../ql/metadata/HiveMaterializedViewsRegistry.java | 12 +-
.../org/apache/hadoop/hive/ql/metadata/Table.java | 40 ++++
.../hive/ql/optimizer/calcite/RelOptHiveTable.java | 206 +++++++++++----------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 88 +++++----
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 57 +++---
5 files changed, 238 insertions(+), 165 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 56047cf..c339258 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -378,9 +378,12 @@ public final class HiveMaterializedViewsRegistry {
List<Interval> intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
rowType = dtFactory.createStructType(druidColTypes, druidColNames);
+ // We can pass null for Hive object because it is only used to retrieve tables
+ // if constraints on a table object are existing, but constraints cannot be defined
+ // for materialized views.
RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName,
rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(),
- conf, new HashMap<>(), new HashMap<>(), new AtomicInteger());
+ conf, null, new HashMap<>(), new HashMap<>(), new HashMap<>(), new AtomicInteger());
DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,
intervals, null, null);
@@ -389,10 +392,13 @@ public final class HiveMaterializedViewsRegistry {
tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE),
optTable, druidTable, ImmutableList.<RelNode>of(scan), ImmutableMap.of());
} else {
- // Build Hive Table Scan Rel
+ // Build Hive Table Scan Rel.
+ // We can pass null for Hive object because it is only used to retrieve tables
+ // if constraints on a table object are existing, but constraints cannot be defined
+ // for materialized views.
RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName,
rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(),
- conf, new HashMap<>(), new HashMap<>(), new AtomicInteger());
+ conf, null, new HashMap<>(), new HashMap<>(), new HashMap<>(), new AtomicInteger());
tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable,
viewTable.getTableName(), null, false, false);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index ae030d9..aba3035 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -112,6 +112,12 @@ public class Table implements Serializable {
* a materialization is up-to-date or not. */
private transient Boolean outdatedForRewritingMaterializedView;
+ /** Constraint related objects */
+ private transient PrimaryKeyInfo pki;
+ private transient ForeignKeyInfo fki;
+ private transient UniqueConstraint uki;
+ private transient NotNullConstraint nnc;
+
/**
* Used only for serialization.
*/
@@ -1109,6 +1115,40 @@ public class Table implements Serializable {
return outdatedForRewritingMaterializedView;
}
+ /* These are only populated during optimization */
+ public PrimaryKeyInfo getPrimaryKeyInfo() {
+ return pki;
+ }
+
+ public void setPrimaryKeyInfo(PrimaryKeyInfo pki) {
+ this.pki = pki;
+ }
+
+ public ForeignKeyInfo getForeignKeyInfo() {
+ return fki;
+ }
+
+ public void setForeignKeyInfo(ForeignKeyInfo fki) {
+ this.fki = fki;
+ }
+
+ public UniqueConstraint getUniqueKeyInfo() {
+ return uki;
+ }
+
+ public void setUniqueKeyInfo(UniqueConstraint uki) {
+ this.uki = uki;
+ }
+
+ public NotNullConstraint getNotNullConstraint() {
+ return nnc;
+ }
+
+ public void setNotNullConstraint(NotNullConstraint nnc) {
+ this.nnc = nnc;
+ }
+
+
public ColumnStatistics getColStats() {
return tTable.isSetColStats() ? tTable.getColStats() : null;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
index f65cc25..53eeb0c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
@@ -49,6 +49,7 @@ import org.apache.calcite.schema.ColumnStrategy;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.mapping.IntPair;
+import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -86,14 +87,15 @@ public class RelOptHiveTable implements RelOptTable {
//~ Instance fields --------------------------------------------------------
- private final RelOptSchema schema;
- private final RelDataTypeFactory typeFactory;
- private final RelDataType rowType;
- private final List<String> qualifiedTblName;
- private final String name;
+ private final RelOptSchema schema;
+ private final RelDataTypeFactory typeFactory;
+ private final RelDataType rowType;
+ private final List<String> qualifiedTblName;
+ private final String name;
private final Table hiveTblMetadata;
private final ImmutableList<ColumnInfo> hiveNonPartitionCols;
private final ImmutableList<ColumnInfo> hivePartitionCols;
+ private final Map<Integer, ColStatistics> hiveColStatsMap;
private final ImmutableMap<Integer, ColumnInfo> hiveNonPartitionColsMap;
private final ImmutableMap<Integer, ColumnInfo> hivePartitionColsMap;
private final ImmutableList<VirtualColumn> hiveVirtualCols;
@@ -101,20 +103,23 @@ public class RelOptHiveTable implements RelOptTable {
private final List<ImmutableBitSet> keys;
private final List<ImmutableBitSet> nonNullablekeys;
private final List<RelReferentialConstraint> referentialConstraints;
- final HiveConf hiveConf;
+ private final HiveConf hiveConf;
+
+ private final Hive db;
+ private final Map<String, Table> tablesCache;
+ private final Map<String, PrunedPartitionList> partitionCache;
+ private final Map<String, ColumnStatsList> colStatsCache;
+ private final AtomicInteger noColsMissingStats;
private double rowCount = -1;
- Map<Integer, ColStatistics> hiveColStatsMap = new HashMap<>();
PrunedPartitionList partitionList;
- Map<String, PrunedPartitionList> partitionCache;
- Map<String, ColumnStatsList> colStatsCache;
- AtomicInteger noColsMissingStats;
protected static final Logger LOG = LoggerFactory.getLogger(RelOptHiveTable.class.getName());
+
public RelOptHiveTable(RelOptSchema calciteSchema, RelDataTypeFactory typeFactory, List<String> qualifiedTblName,
- RelDataType rowType, Table hiveTblMetadata, List<ColumnInfo> hiveNonPartitionCols,
- List<ColumnInfo> hivePartitionCols, List<VirtualColumn> hiveVirtualCols, HiveConf hconf,
+ RelDataType rowType, Table hiveTblMetadata, List<ColumnInfo> hiveNonPartitionCols, List<ColumnInfo> hivePartitionCols,
+ List<VirtualColumn> hiveVirtualCols, HiveConf hconf, Hive db, Map<String, Table> tabNameToTabObject,
Map<String, PrunedPartitionList> partitionCache, Map<String, ColumnStatsList> colStatsCache,
AtomicInteger noColsMissingStats) {
this.schema = calciteSchema;
@@ -123,6 +128,7 @@ public class RelOptHiveTable implements RelOptTable {
this.name = this.qualifiedTblName.stream().collect(Collectors.joining("."));
this.rowType = rowType;
this.hiveTblMetadata = hiveTblMetadata;
+ this.hiveColStatsMap = new HashMap<>();
this.hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols);
this.hiveNonPartitionColsMap = HiveCalciteUtil.getColInfoMap(hiveNonPartitionCols, 0);
this.hivePartitionCols = ImmutableList.copyOf(hivePartitionCols);
@@ -130,6 +136,8 @@ public class RelOptHiveTable implements RelOptTable {
this.noOfNonVirtualCols = hiveNonPartitionCols.size() + hivePartitionCols.size();
this.hiveVirtualCols = ImmutableList.copyOf(hiveVirtualCols);
this.hiveConf = hconf;
+ this.db = db;
+ this.tablesCache = tabNameToTabObject;
this.partitionCache = partitionCache;
this.colStatsCache = colStatsCache;
this.noColsMissingStats = noColsMissingStats;
@@ -216,7 +224,8 @@ public class RelOptHiveTable implements RelOptTable {
// 3. Build new Table
return new RelOptHiveTable(this.schema, this.typeFactory, this.qualifiedTblName, newRowType,
this.hiveTblMetadata, newHiveNonPartitionCols, newHivePartitionCols, newHiveVirtualCols,
- this.hiveConf, this.partitionCache, this.colStatsCache, this.noColsMissingStats);
+ this.hiveConf, this.db, this.tablesCache, this.partitionCache, this.colStatsCache,
+ this.noColsMissingStats);
}
// Given a key this method returns true if all of the columns in the key are not nullable
@@ -245,19 +254,14 @@ public class RelOptHiveTable implements RelOptTable {
}
private Pair<List<ImmutableBitSet>, List<ImmutableBitSet>> generateKeys() {
+ final PrimaryKeyInfo primaryKeyInfo = hiveTblMetadata.getPrimaryKeyInfo();
+ final UniqueConstraint uniqueKeyInfo = hiveTblMetadata.getUniqueKeyInfo();
ImmutableList.Builder<ImmutableBitSet> builder = ImmutableList.builder();
ImmutableList.Builder<ImmutableBitSet> nonNullbuilder = ImmutableList.builder();
// First PK
- final PrimaryKeyInfo pki;
- try {
- pki = Hive.get().getReliablePrimaryKeys(
- hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName());
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- if (!pki.getColNames().isEmpty()) {
+ if (primaryKeyInfo != null && !primaryKeyInfo.getColNames().isEmpty()) {
ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
- for (String pkColName : pki.getColNames().values()) {
+ for (String pkColName : primaryKeyInfo.getColNames().values()) {
int pkPos;
for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) {
String colName = rowType.getFieldNames().get(pkPos);
@@ -275,103 +279,107 @@ public class RelOptHiveTable implements RelOptTable {
nonNullbuilder.add(key);
}
// Then UKs
- final UniqueConstraint uki;
- try {
- uki = Hive.get().getReliableUniqueConstraints(
- hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName());
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- for (List<UniqueConstraintCol> ukCols : uki.getUniqueConstraints().values()) {
- ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
- boolean isNonNullable = true;
- for (UniqueConstraintCol ukCol : ukCols) {
- int ukPos;
- for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) {
- String colName = rowType.getFieldNames().get(ukPos);
- if (ukCol.colName.equals(colName)) {
- if(rowType.getFieldList().get(ukPos).getType().isNullable()) {
- // they should all be nullable
- isNonNullable = false;
+ if (uniqueKeyInfo != null && !uniqueKeyInfo.getUniqueConstraints().isEmpty()) {
+ for (List<UniqueConstraintCol> ukCols : uniqueKeyInfo.getUniqueConstraints().values()) {
+ ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
+ boolean isNonNullable = true;
+ for (UniqueConstraintCol ukCol : ukCols) {
+ int ukPos;
+ for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) {
+ String colName = rowType.getFieldNames().get(ukPos);
+ if (ukCol.colName.equals(colName)) {
+ if (rowType.getFieldList().get(ukPos).getType().isNullable()) {
+ // they should all be nullable
+ isNonNullable = false;
+ }
+ break;
}
- break;
}
+ if (ukPos == rowType.getFieldNames().size()) {
+ LOG.error("Column for unique constraint definition " + ukCol.colName + " not found");
+ }
+ keys.set(ukPos);
}
- if (ukPos == rowType.getFieldNames().size()) {
- LOG.error("Column for unique constraint definition " + ukCol.colName + " not found");
+ ImmutableBitSet key = keys.build();
+ builder.add(key);
+ if (isNonNullable) {
+ nonNullbuilder.add(key);
}
- keys.set(ukPos);
- }
- ImmutableBitSet key = keys.build();
- builder.add(key);
- if(isNonNullable) {
- nonNullbuilder.add(key);
}
}
return new Pair<>(builder.build(), nonNullbuilder.build());
}
private List<RelReferentialConstraint> generateReferentialConstraints() {
- final ForeignKeyInfo fki;
- try {
- fki = Hive.get().getReliableForeignKeys(
- hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName());
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
+ final ForeignKeyInfo foreignKeyInfo = hiveTblMetadata.getForeignKeyInfo();
ImmutableList.Builder<RelReferentialConstraint> builder = ImmutableList.builder();
- for (List<ForeignKeyCol> fkCols : fki.getForeignKeys().values()) {
- List<String> foreignKeyTableQualifiedName = qualifiedTblName;
- String parentDatabaseName = fkCols.get(0).parentDatabaseName;
- String parentTableName = fkCols.get(0).parentTableName;
- List<String> parentTableQualifiedName = new ArrayList<>();
- if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) {
- parentTableQualifiedName.add(parentDatabaseName);
- }
- parentTableQualifiedName.add(parentTableName);
- Table parentTab = null;
- try {
- // TODO: We have a cache for Table objects in SemanticAnalyzer::getTableObjectByName()
- // We need to move that cache elsewhere and use it from places like this.
- parentTab = Hive.get().getTable(parentDatabaseName, parentTableName);
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- if (parentTab == null) {
- LOG.error("Table for primary key not found: "
- + "databaseName: " + parentDatabaseName+ ", "
+ if (foreignKeyInfo != null && !foreignKeyInfo.getForeignKeys().isEmpty()) {
+ for (List<ForeignKeyCol> fkCols : foreignKeyInfo.getForeignKeys().values()) {
+ String parentDatabaseName = fkCols.get(0).parentDatabaseName;
+ String parentTableName = fkCols.get(0).parentTableName;
+ String qualifiedName;
+ List<String> parentTableQualifiedName = new ArrayList<>();
+ if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) {
+ parentTableQualifiedName.add(parentDatabaseName);
+ parentTableQualifiedName.add(parentTableName);
+ qualifiedName = TableName.getDbTable(
+ parentDatabaseName, parentTableName);
+ } else {
+ parentTableQualifiedName.add(parentTableName);
+ qualifiedName = parentTableName;
+ }
+ Table parentTab = getTable(qualifiedName);
+ if (parentTab == null) {
+ LOG.error("Table for primary key not found: "
+ + "databaseName: " + parentDatabaseName + ", "
+ "tableName: " + parentTableName);
- return ImmutableList.of();
- }
- ImmutableList.Builder<IntPair> keys = ImmutableList.builder();
- for (ForeignKeyCol fkCol : fkCols) {
- int fkPos;
- for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) {
- String fkColName = rowType.getFieldNames().get(fkPos);
- if (fkColName.equals(fkCol.childColName)) {
- break;
- }
+ return ImmutableList.of();
}
- int pkPos;
- for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) {
- String pkColName = parentTab.getAllCols().get(pkPos).getName();
- if (pkColName.equals(fkCol.parentColName)) {
- break;
+ ImmutableList.Builder<IntPair> keys = ImmutableList.builder();
+ for (ForeignKeyCol fkCol : fkCols) {
+ int fkPos;
+ for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) {
+ String fkColName = rowType.getFieldNames().get(fkPos);
+ if (fkColName.equals(fkCol.childColName)) {
+ break;
+ }
}
+ int pkPos;
+ for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) {
+ String pkColName = parentTab.getAllCols().get(pkPos).getName();
+ if (pkColName.equals(fkCol.parentColName)) {
+ break;
+ }
+ }
+ if (fkPos == rowType.getFieldNames().size()
+ || pkPos == parentTab.getAllCols().size()) {
+ LOG.error("Column for foreign key definition " + fkCol + " not found");
+ return ImmutableList.of();
+ }
+ keys.add(IntPair.of(fkPos, pkPos));
}
- if (fkPos == rowType.getFieldNames().size()
- || pkPos == parentTab.getAllCols().size()) {
- LOG.error("Column for foreign key definition " + fkCol + " not found");
- return ImmutableList.of();
- }
- keys.add(IntPair.of(fkPos, pkPos));
+ builder.add(RelReferentialConstraintImpl.of(qualifiedTblName,
+ parentTableQualifiedName, keys.build()));
}
- builder.add(RelReferentialConstraintImpl.of(foreignKeyTableQualifiedName,
- parentTableQualifiedName, keys.build()));
}
return builder.build();
}
+ private Table getTable(String tableName) {
+ if (!tablesCache.containsKey(tableName)) {
+ try {
+ Table table = db.getTable(tableName);
+ if (table != null) {
+ tablesCache.put(tableName, table);
+ }
+ return table;
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return tablesCache.get(tableName);
+ }
+
@Override
public RelNode toRel(ToRelContext context) {
return new LogicalTableScan(context.getCluster(), this);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 4011d99..0796bc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -72,7 +72,6 @@ import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
-import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
@@ -135,7 +134,6 @@ import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.log.PerfLogger;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo;
@@ -2996,7 +2994,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
intervals, null, null);
optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName,
rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf,
- partitionCache, colStatsCache, noColsMissingStats);
+ db, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats);
final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias,
getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf,
@@ -3007,7 +3005,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
} else {
optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName,
rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf,
- partitionCache, colStatsCache, noColsMissingStats);
+ db, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats);
final HiveTableScan hts = new HiveTableScan(cluster,
cluster.traitSetOf(HiveRelNode.CONVENTION), optTable,
null == tableAlias ? tabMetaData.getTableName() : tableAlias,
@@ -3053,7 +3051,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
fullyQualifiedTabName.add(tabMetaData.getTableName());
optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName,
rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf,
- partitionCache, colStatsCache, noColsMissingStats);
+ db, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats);
// Build Hive Table Scan Rel
tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable,
null == tableAlias ? tabMetaData.getTableName() : tableAlias,
@@ -3083,45 +3081,46 @@ public class CalcitePlanner extends SemanticAnalyzer {
private RelDataType inferNotNullableColumns(Table tabMetaData, RelDataType rowType)
throws HiveException {
- // Retrieve not null constraints
- final NotNullConstraint nnc = Hive.get().getReliableNotNullConstraints(
- tabMetaData.getDbName(), tabMetaData.getTableName());
- // Retrieve primary key constraints (cannot be null)
- final PrimaryKeyInfo pkc = Hive.get().getReliablePrimaryKeys(
- tabMetaData.getDbName(), tabMetaData.getTableName());
- if (nnc.getNotNullConstraints().isEmpty() && pkc.getColNames().isEmpty()) {
+ final NotNullConstraint nnc = tabMetaData.getNotNullConstraint();
+ final PrimaryKeyInfo pkc = tabMetaData.getPrimaryKeyInfo();
+ if ((nnc == null || nnc.getNotNullConstraints().isEmpty()) &&
+ (pkc == null || pkc.getColNames().isEmpty())) {
return rowType;
}
// Build the bitset with not null columns
ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
- for (String nnCol : nnc.getNotNullConstraints().values()) {
- int nnPos = -1;
- for (int i = 0; i < rowType.getFieldNames().size(); i++) {
- if (rowType.getFieldNames().get(i).equals(nnCol)) {
- nnPos = i;
- break;
+ if (nnc != null) {
+ for (String nnCol : nnc.getNotNullConstraints().values()) {
+ int nnPos = -1;
+ for (int i = 0; i < rowType.getFieldNames().size(); i++) {
+ if (rowType.getFieldNames().get(i).equals(nnCol)) {
+ nnPos = i;
+ break;
+ }
}
+ if (nnPos == -1) {
+ LOG.error("Column for not null constraint definition " + nnCol + " not found");
+ return rowType;
+ }
+ builder.set(nnPos);
}
- if (nnPos == -1) {
- LOG.error("Column for not null constraint definition " + nnCol + " not found");
- return rowType;
- }
- builder.set(nnPos);
}
- for (String pkCol : pkc.getColNames().values()) {
- int pkPos = -1;
- for (int i = 0; i < rowType.getFieldNames().size(); i++) {
- if (rowType.getFieldNames().get(i).equals(pkCol)) {
- pkPos = i;
- break;
+ if (pkc != null) {
+ for (String pkCol : pkc.getColNames().values()) {
+ int pkPos = -1;
+ for (int i = 0; i < rowType.getFieldNames().size(); i++) {
+ if (rowType.getFieldNames().get(i).equals(pkCol)) {
+ pkPos = i;
+ break;
+ }
}
+ if (pkPos == -1) {
+ LOG.error("Column for not null constraint definition " + pkCol + " not found");
+ return rowType;
+ }
+ builder.set(pkPos);
}
- if (pkPos == -1) {
- LOG.error("Column for not null constraint definition " + pkCol + " not found");
- return rowType;
- }
- builder.set(pkPos);
}
ImmutableBitSet bitSet = builder.build();
@@ -5172,6 +5171,27 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
}
+ @Override
+ protected Table getTableObjectByName(String tableName, boolean throwException) throws HiveException {
+ if (!tabNameToTabObject.containsKey(tableName)) {
+ // TODO: The code below should be a single HMS call and possibly unified with method in SemanticAnalyzer
+ Table table = db.getTable(tableName, throwException);
+ if (table != null) {
+ table.setPrimaryKeyInfo(db.getReliablePrimaryKeys(
+ table.getDbName(), table.getTableName()));
+ table.setForeignKeyInfo(db.getReliableForeignKeys(
+ table.getDbName(), table.getTableName()));
+ table.setUniqueKeyInfo(db.getReliableUniqueConstraints(
+ table.getDbName(), table.getTableName()));
+ table.setNotNullConstraint(db.getReliableNotNullConstraints(
+ table.getDbName(), table.getTableName()));
+ tabNameToTabObject.put(tableName, table);
+ }
+ return table;
+ }
+ return tabNameToTabObject.get(tableName);
+ }
+
/**
* This method can be called at startup time to pre-register all the
* additional Hive classes (compared to Calcite core classes) that may
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 808c5c1..075df65 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -1825,7 +1825,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
Table table = null;
try {
- table = this.getTableObjectByName(tableName);
+ table = getTableObjectByName(tableName);
} catch (HiveException ex) {
throw new SemanticException(ex);
}
@@ -2104,8 +2104,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Get table details from tabNameToTabObject cache
Table tab = getTableObjectByName(tabName, false);
if (tab != null) {
- // do a deep copy, in case downstream changes it.
- tab = new Table(tab.getTTable().deepCopy());
+ // copy table object in case downstream changes it
+ Table newTab = new Table(tab.getTTable().deepCopy());
+ // copy constraints, we do not need deep copy as
+ // they should not be changed
+ newTab.setPrimaryKeyInfo(tab.getPrimaryKeyInfo());
+ newTab.setForeignKeyInfo(tab.getForeignKeyInfo());
+ newTab.setUniqueKeyInfo(tab.getUniqueKeyInfo());
+ newTab.setNotNullConstraint(tab.getNotNullConstraint());
+ tab = newTab;
}
if (tab == null ||
tab.getDbName().equals(SessionState.get().getCurrentDatabase())) {
@@ -11996,8 +12003,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return rewrittenQuery;
}
- private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set<String> cteAlias,
- Context ctx, Hive db, Map<String, Table> tabNameToTabObject, Set<Integer> ignoredTokens)
+ private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set<String> cteAlias,
+ Context ctx, Hive db, Map<String, Table> tabNameToTabObject, Set<Integer> ignoredTokens)
throws SemanticException {
Queue<Node> queue = new LinkedList<>();
queue.add(ast);
@@ -12039,9 +12046,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
continue;
}
- Table table = getTable(db, tabIdName, tabNameToTabObject);
+ Table table = null;
+ try {
+ table = getTableObjectByName(tabIdName, false);
+ } catch (HiveException e) {
+ // This should not happen.
+ throw new SemanticException("Got exception though getTableObjectByName method should ignore it");
+ }
if (table == null) {
// Table may not be found when materialization of CTE is on.
+ STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
continue;
}
@@ -12049,9 +12063,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// When we are querying a materialized view directly, we check whether the source tables
// do not apply any policies.
for (String qName : table.getCreationMetadata().getTablesUsed()) {
- table = getTable(db, qName, tabNameToTabObject);
- if (table == null) {
- // This should not happen
+ try {
+ table = getTableObjectByName(qName, true);
+ } catch (HiveException e) {
+ // This should not happen.
throw new SemanticException("Table " + qName + " not found when trying to obtain it to check masking/filtering " +
"policies");
}
@@ -12106,22 +12121,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- private static Table getTable(Hive db, String tabIdName, Map<String, Table> tabNameToTabObject) {
- Table table = null;
- try {
- if (!tabNameToTabObject.containsKey(tabIdName)) {
- table = db.getTable(tabIdName, true);
- tabNameToTabObject.put(tabIdName, table);
- } else {
- table = tabNameToTabObject.get(tabIdName);
- }
- } catch (HiveException e) {
- // Table may not be found when materialization of CTE is on.
- STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
- }
- return table;
- }
-
private static void extractColumnInfos(Table table, List<String> colNames, List<String> colTypes) {
for (FieldSchema col : table.getAllCols()) {
colNames.add(col.getName());
@@ -12134,8 +12133,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// the table needs to be masked or filtered.
// For the replacement, we leverage the methods that are used for
// unparseTranslator.
- protected static ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, TokenRewriteStream tokenRewriteStream,
- Context ctx, Hive db, Map<String, Table> tabNameToTabObject, Set<Integer> ignoredTokens)
+ protected ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, TokenRewriteStream tokenRewriteStream,
+ Context ctx, Hive db, Map<String, Table> tabNameToTabObject, Set<Integer> ignoredTokens)
throws SemanticException {
// 1. collect information about CTE if there is any.
// The base table of CTE should be masked.
@@ -13809,7 +13808,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (DUMMY_TABLE.equals(alias)) {
continue;
}
- Table table = this.getTableObjectByName(qb.getTabNameForAlias(alias));
+ Table table = getTableObjectByName(qb.getTabNameForAlias(alias));
if (table.isTemporary()) {
throw new SemanticException("View definition references temporary table " + alias);
}
@@ -14039,7 +14038,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
String tableName = getUnescapedName((ASTNode) tree.getChild(0).getChild(0));
Table tbl;
try {
- tbl = this.getTableObjectByName(tableName);
+ tbl = getTableObjectByName(tableName);
} catch (InvalidTableException e) {
throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName), e);
}
[hive] 02/02: HIVE-21834: Avoid unnecessary calls to simplify
filter conditions (Jesus Camacho Rodriguez, reviewed by Gopal V)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 51b1c6b55539fb97064f1db566886d9ddb6642fd
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Wed Jun 5 10:11:49 2019 -0700
HIVE-21834: Avoid unnecessary calls to simplify filter conditions (Jesus Camacho Rodriguez, reviewed by Gopal V)
---
.../org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java | 4 ++--
.../test/results/clientpositive/llap/external_jdbc_table_perf.q.out | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java
index 4794883..f50779d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java
@@ -92,8 +92,8 @@ public class HiveRelBuilder extends RelBuilder {
@Override
public RelBuilder filter(Iterable<? extends RexNode> predicates) {
- final RexNode x = RexUtil.simplify(cluster.getRexBuilder(),
- RexUtil.composeConjunction(cluster.getRexBuilder(), predicates, false));
+ final RexNode x = RexUtil.composeConjunction(
+ cluster.getRexBuilder(), predicates, false);
if (!x.isAlwaysTrue()) {
final RelNode input = build();
final RelNode filter = HiveRelFactories.HIVE_FILTER_FACTORY.createFilter(input, x);
diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out
index c1314c6..a394f2b 100644
--- a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out
+++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out
@@ -2256,7 +2256,7 @@ FROM "STORE_SALES"
WHERE "ss_customer_sk" IS NOT NULL AND "ss_sold_date_sk" IS NOT NULL) AS "t0"
INNER JOIN (SELECT "d_date_sk"
FROM "DATE_DIM"
-WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ss_sold_date_sk" = "t2"."d_date_sk"
+WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ss_sold_date_sk" = "t2"."d_date_sk"
hive.sql.query.fieldNames ss_customer_sk0
hive.sql.query.fieldTypes int
hive.sql.query.split false
@@ -2289,7 +2289,7 @@ FROM "WEB_SALES"
WHERE "ws_bill_customer_sk" IS NOT NULL AND "ws_sold_date_sk" IS NOT NULL) AS "t0"
INNER JOIN (SELECT "d_date_sk"
FROM "DATE_DIM"
-WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ws_sold_date_sk" = "t2"."d_date_sk"
+WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ws_sold_date_sk" = "t2"."d_date_sk"
GROUP BY "t0"."ws_bill_customer_sk"
hive.sql.query.fieldNames ws_bill_customer_sk0,$f1
hive.sql.query.fieldTypes int,boolean
@@ -2318,7 +2318,7 @@ FROM "CATALOG_SALES"
WHERE "cs_ship_customer_sk" IS NOT NULL AND "cs_sold_date_sk" IS NOT NULL) AS "t0"
INNER JOIN (SELECT "d_date_sk"
FROM "DATE_DIM"
-WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."cs_sold_date_sk" = "t2"."d_date_sk"
+WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."cs_sold_date_sk" = "t2"."d_date_sk"
GROUP BY "t0"."cs_ship_customer_sk"
hive.sql.query.fieldNames cs_ship_customer_sk0,$f1
hive.sql.query.fieldTypes int,boolean