You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@hive.apache.org by "ngsg (via GitHub)" <gi...@apache.org> on 2026/07/06 12:58:53 UTC
Re: [PR] HIVE-29672: Extract column stats, constraints, token, and WM logic from ObjectStore [hive]
ngsg commented on code in PR #6552:
URL: https://github.com/apache/hive/pull/6552#discussion_r3528971713
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java:
##########
@@ -47,8 +47,8 @@
@VisibleForTesting
public abstract class GetHelper<A, T> {
private static final Logger LOG = LoggerFactory.getLogger(GetHelper.class);
- private static final Counter directSqlErrors = Metrics.getRegistry() != null ?
- Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS) : new Counter();
+ private static Counter directSqlErrors = setDirectSqlErrors(Metrics.getRegistry() != null ?
Review Comment:
nit, Could we initialize the field directly instead of using `setDirectSqlErrors`?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java:
##########
@@ -61,11 +61,11 @@ public abstract class GetHelper<A, T> {
private boolean success = false;
protected T results = null;
- public GetHelper(RawStoreAware rsa, A args) throws MetaException {
+ public GetHelper(RawStoreBundle rsa, A args) throws MetaException {
Review Comment:
nit, Could we rename `rsa` to `rsb`, `bundle`, or similar?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java:
##########
@@ -23,17 +23,17 @@
import org.apache.hadoop.hive.metastore.api.MetaException;
public abstract class GetListHelper<A, T> extends GetHelper<A, List<T>> {
- public GetListHelper(RawStoreAware rsa, A args) throws MetaException {
+ public GetListHelper(RawStoreBundle rsa, A args) throws MetaException {
Review Comment:
nit, Could we rename `rsa` to `rsb`, `bundle`, or similar?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java:
##########
@@ -0,0 +1,1226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import javax.jdo.Query;
+import javax.jdo.datastore.JDOConnection;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.Batchable;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.Deadline;
+import org.apache.hadoop.hive.metastore.PersistenceManagerProvider;
+import org.apache.hadoop.hive.metastore.QueryWrapper;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionFilterMode;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats;
+import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql;
+import org.apache.hadoop.hive.metastore.metastore.GetHelper;
+import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
+import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
+import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore;
+import org.apache.hadoop.hive.metastore.metastore.iface.TableStore;
+import org.apache.hadoop.hive.metastore.model.MPartition;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MTable;
+import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.utils.RetryingExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING;
+import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams;
+import static org.apache.hadoop.hive.metastore.ObjectStore.isCurrentStatsValidForTheQuery;
+import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers;
+
+public class ColStatsStoreImpl extends RawStoreBundle implements ColStatsStore {
+ private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class);
+
+ private DatabaseProduct dbType;
+ protected int batchSize = NO_BATCHING;
+ private boolean areTxnStatsSupported = false;
+ private Configuration conf;
+ private String schema;
+ private SQLGenerator sqlGenerator;
+
+ @Override
+ public void setBaseStore(RawStore store) {
+ super.setBaseStore(store);
+ this.dbType = PersistenceManagerProvider.getDatabaseProduct();
+ this.batchSize = MetastoreConf.getIntVar(store.getConf(),
+ MetastoreConf.ConfVars.RAWSTORE_PARTITION_BATCH_SIZE);
+ this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(),
+ MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED);
+ this.conf = store.getConf();
+ this.sqlGenerator = new SQLGenerator(dbType, conf);
+ String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema");
+ this.schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null);
+ }
+
+ @Override
+ public List<TableName> getTableNamesWithStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName> (this, null) {
Review Comment:
nit, `new GetListHelper<TableName, TableName> (this, null)` has an extra space before `(`.
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java:
##########
@@ -0,0 +1,1226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import javax.jdo.Query;
+import javax.jdo.datastore.JDOConnection;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.Batchable;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.Deadline;
+import org.apache.hadoop.hive.metastore.PersistenceManagerProvider;
+import org.apache.hadoop.hive.metastore.QueryWrapper;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionFilterMode;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats;
+import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql;
+import org.apache.hadoop.hive.metastore.metastore.GetHelper;
+import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
+import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
+import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore;
+import org.apache.hadoop.hive.metastore.metastore.iface.TableStore;
+import org.apache.hadoop.hive.metastore.model.MPartition;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MTable;
+import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.utils.RetryingExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING;
+import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams;
+import static org.apache.hadoop.hive.metastore.ObjectStore.isCurrentStatsValidForTheQuery;
+import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers;
+
+public class ColStatsStoreImpl extends RawStoreBundle implements ColStatsStore {
+ private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class);
+
+ private DatabaseProduct dbType;
+ protected int batchSize = NO_BATCHING;
+ private boolean areTxnStatsSupported = false;
+ private Configuration conf;
+ private String schema;
+ private SQLGenerator sqlGenerator;
+
+ @Override
+ public void setBaseStore(RawStore store) {
+ super.setBaseStore(store);
+ this.dbType = PersistenceManagerProvider.getDatabaseProduct();
+ this.batchSize = MetastoreConf.getIntVar(store.getConf(),
+ MetastoreConf.ConfVars.RAWSTORE_PARTITION_BATCH_SIZE);
+ this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(),
+ MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED);
+ this.conf = store.getConf();
+ this.sqlGenerator = new SQLGenerator(dbType, conf);
+ String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema");
+ this.schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null);
+ }
+
+ @Override
+ public List<TableName> getTableNamesWithStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName> (this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getTableNamesWithStats();
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+ }.run(false);
+ }
+
+ @Override
+ public Map<String, List<String>> getPartitionColsWithStats(TableName tableName)
+ throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new GetHelper<TableName, Map<String, List<String>>>(this, null) {
+ @Override
+ protected Map<String, List<String>> getSqlResult() throws MetaException {
+ try {
+ return getDirectSql().getColAndPartNamesWithStats(catName, dbName, tblName);
+ } catch (Throwable ex) {
+ LOG.error("DirectSQL failed", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, List<String>> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+
+ @Override
+ protected String describeResult() {
+ return results.size() + " partitions";
+ }
+ }.run(false);
+ }
+
+ @Override
+ public List<TableName> getAllTableNamesForStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName>(this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getAllTableNamesForStats();
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ List<TableName> result = new ArrayList<>();
+ String paramStr = "", whereStr = "";
+ for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) {
+ if (i != 0) {
+ paramStr += ", ";
+ whereStr += "||";
+ }
+ paramStr += "java.lang.String tt" + i;
+ whereStr += " tableType == tt" + i;
+ }
+ Query query = pm.newQuery(MTable.class, whereStr);
+ query.declareParameters(paramStr);
+ Collection<MTable> tbls =
+ (Collection<MTable>) query.executeWithArray(MetaStoreDirectSql.STATS_TABLE_TYPES);
+ pm.retrieveAll(tbls);
+ for (MTable tbl : tbls) {
+ result.add(new TableName(
+ tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName()));
+ }
+ return result;
+ }
+ }.run(false);
+ }
+
+ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj,
+ MTableColumnStatistics oldStats) throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ String colName = mStatsObj.getColName();
+
+ LOG.info("Updating table level column statistics for table={} colName={}",
+ Warehouse.getCatalogQualifiedTableName(table), colName);
+ validateTableCols(table, Lists.newArrayList(colName));
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ private void writeMPartitionColumnStatistics(Table table, Partition partition,
+ MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) {
+ String catName = mStatsObj.getPartition().getTable().getDatabase().getCatalogName();
+ String dbName = mStatsObj.getPartition().getTable().getDatabase().getName();
+ String tableName = mStatsObj.getPartition().getTable().getTableName();
+ String partName = mStatsObj.getPartition().getPartitionName();
+ String colName = mStatsObj.getColName();
+
+ Preconditions.checkState(this.baseStore.isActiveTransaction());
+
+ LOG.info("Updating partition level column statistics for table=" +
+ TableName.getQualified(catName, dbName, tableName) +
+ " partName=" + partName + " colName=" + colName);
+
+ boolean foundCol = false;
+ List<FieldSchema> colList = partition.getSd().getCols();
+ for (FieldSchema col : colList) {
+ if (col.getName().equals(mStatsObj.getColName())) {
+ foundCol = true;
+ break;
+ }
+ }
+
+ if (!foundCol) {
+ LOG.warn("Column " + colName + " for which stats gathering is requested doesn't exist.");
+ }
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ @Override
+ public Map<String, String> updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ long start = System.currentTimeMillis();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ // DataNucleus objects get detached all over the place for no (real) reason.
+ // So let's not use them anywhere unless absolutely necessary.
+ MTable mTable = baseStore.ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName());
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> transactionHelper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ transactionHelper.setTransactionSavePoint(savePoint);
+ executePlainSQL(
+ sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()),
+ true,
+ exception -> {
+ transactionHelper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mTable);
+ Table table = baseStore.unwrap(TableStore.class).getTable(new TableName(catName, statsDesc.getDbName(),
+ statsDesc.getTableName()), null, -1);
+ List<String> colNames = new ArrayList<>();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+
+ Map<String, MTableColumnStatistics> oldStats = Maps.newHashMap();
+ List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, colStats.getEngine());
+ for (MTableColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc,
+ statsObj, colStats.getEngine());
+ writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName()));
+ // There is no need to add colname again, otherwise we will get duplicate colNames.
+ }
+
+ // Set the table properties
+ // No need to check again if it exists.
+ String dbname = table.getDbName();
+ String name = table.getTableName();
+ MTable oldt = mTable;
+ Map<String, String> newParams = new HashMap<>(table.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters());
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams,
+ writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(oldt.getParameters(), oldt.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name);
+ }
+ oldt.setWriteId(writeId);
+ }
+ }
+ oldt.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+
+ LOG.debug("{} updateTableColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ long start = System.currentTimeMillis();
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ MPartition mPartition =
+ baseStore.ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals);
+ if (mPartition == null) {
+ throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
+ }
+
+ List<String> colNames = new ArrayList<>();
+ for(ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> helper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ helper.setTransactionSavePoint(savePoint);
+ executePlainSQL(sqlGenerator.addForUpdateNoWait(
+ "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()),
+ true,
+ exception -> {
+ helper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mPartition);
+ Partition partition = baseStore.unwrap(TableStore.class)
+ .getPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getValues(), null);
+ Map<String, MPartitionColumnStatistics> oldStats = Maps.newHashMap();
+ List<MPartitionColumnStatistics> stats =
+ getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine());
+ for (MPartitionColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition,
+ statsDesc, statsObj, colStats.getEngine());
+ writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName()));
+ }
+
+ Map<String, String> newParams = new HashMap<>(mPartition.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(table);
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(
+ TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(),
+ newParams, writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName());
+ }
+ mPartition.setWriteId(writeId);
+ }
+ }
+ mPartition.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+ LOG.debug("{} updatePartitionColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
+ Map<String, ColumnStatistics> partColStatsMap,
+ Table tbl,
+ List<TransactionalMetaStoreEventListener> listeners,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+
+ return new GetHelper<TableName, Map<String, Map<String, String>>>(this, null) {
+ @Override
+ protected String describeResult() {
+ return "Map of partition key to column stats if successful";
+ }
+ @Override
+ protected Map<String, Map<String, String>> getSqlResult()
+ throws MetaException {
+ return getDirectSql().updatePartitionColumnStatisticsBatch(partColStatsMap, tbl,
+ listeners, validWriteIds, writeId);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, Map<String, String>> getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled");
+ }
+ }.run(false);
+ }
+
+ private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, String engine)
+ throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ if (colNames.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ validateTableCols(table, colNames);
+
+ List<MTableColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ result =
+ Batchable.runBatched(batchSize, colNames, new Batchable<String, MTableColumnStatistics>() {
+ @Override
+ public List<MTableColumnStatistics> run(List<String> input)
+ throws MetaException {
+ StringBuilder filter =
+ new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && (");
+ StringBuilder paramStr = new StringBuilder(
+ "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4");
+ Object[] params = new Object[input.size() + 4];
+ params[0] = table.getTableName();
+ params[1] = table.getDbName();
+ params[2] = table.getCatName();
+ params[3] = engine;
+ for (int i = 0; i < input.size(); ++i) {
+ filter.append((i == 0) ? "" : " || ").append("colName == c").append(i);
+ paramStr.append(", java.lang.String c").append(i);
+ params[i + 4] = input.get(i);
+ }
+ filter.append(")");
+ query.setFilter(filter.toString());
+ query.declareParameters(paramStr.toString());
+ List<MTableColumnStatistics> paritial = (List<MTableColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(paritial);
+ return paritial;
+ }
+ });
+
+ if (result.size() > colNames.size()) {
+ throw new MetaException("Unexpected " + result.size() + " statistics for "
+ + colNames.size() + " columns");
+ }
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public List<ColumnStatistics> getTableColumnStatistics(
+ TableName tblName,
+ List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<ColumnStatistics> result = new ArrayList<>();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ ColumnStatistics cs = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ return getTableColumnStatisticsInternal(tableName, colNames, engine);
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine,
+ String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, set No to the compliance flag.
+ Boolean isCompliant = null;
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ if (writeIdList != null) {
+ MTable table = baseStore.ensureGetMTable(catName, dbName, tblName);
+ isCompliant = !TxnUtils.isTransactionalTable(table.getParameters())
+ || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table.getParameters(), table.getWriteId(), writeIdList, false));
+ }
+ ColumnStatistics stats = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tblName), colNames, engine);
+ if (stats != null && isCompliant != null) {
+ stats.setIsStatsCompliant(isCompliant);
+ }
+ return stats;
+ }
+
+ protected ColumnStatistics getTableColumnStatisticsInternal(
+ TableName tableName, final List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetStatHelper(tableName, this) {
+ @Override
+ protected ColumnStatistics getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected ColumnStatistics getJdoResult() throws MetaException {
+
+ List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, engine);
+ if (mStats.isEmpty()) {
+ return null;
+ }
+ // LastAnalyzed is stored per column, but thrift object has it per
+ // multiple columns. Luckily, nobody actually uses it, so we will set to
+ // lowest value of all columns for now.
+ ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
+ List<ColumnStatisticsObj> statObjs = new ArrayList<>(mStats.size());
+ for (MTableColumnStatistics mStat : mStats) {
+ if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
+ desc.setLastAnalyzed(mStat.getLastAnalyzed());
+ }
+ statObjs.add(StatObjectConverter.getColumnStatisticsObj(mStat, enableBitVector, enableKll));
+ Deadline.checkTimeout();
+ }
+ ColumnStatistics colStat = new ColumnStatistics(desc, statObjs);
+ colStat.setEngine(engine);
+ return colStat;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<List<ColumnStatistics>> getPartitionColumnStatistics(TableName tblName,
+ List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<List<ColumnStatistics>> result = new ArrayList<>();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ List<ColumnStatistics> cs = getPartitionColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), partNames, colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(TableName tableName,
+ List<String> partNames, List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ return getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ TableName tableName,
+ List<String> partNames, List<String> colNames,
+ String engine, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ List<ColumnStatistics> allStats = getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ if (writeIdList != null) {
+ if (!areTxnStatsSupported) {
+ for (ColumnStatistics cs : allStats) {
+ cs.setIsStatsCompliant(false);
+ }
+ } else {
+ // TODO: this could be improved to get partitions in bulk
+ for (ColumnStatistics cs : allStats) {
+ MPartition mpart = baseStore.ensureGetMPartition(tableName,
+ Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName()));
+ if (mpart == null
+ || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) {
+ if (mpart != null) {
+ LOG.debug("The current metastore transactional partition column statistics for {} {} "
+ + "(write ID {}) are not valid for current query ({})", tableName,
+ mpart.getPartitionName(), mpart.getWriteId(), writeIdList);
+ }
+ cs.setIsStatsCompliant(false);
+ } else {
+ cs.setIsStatsCompliant(true);
+ }
+ }
+ }
+ }
+ return allStats;
+ }
+
+ protected List<ColumnStatistics> getPartitionColumnStatisticsInternal(
+ TableName tableName, final List<String> partNames, final List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetListHelper<TableName, ColumnStatistics>(this, tableName) {
+ @Override
+ protected List<ColumnStatistics> getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema).getPartitionStats(
+ catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll);
+ }
+ @Override
+ protected List<ColumnStatistics> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ List<MPartitionColumnStatistics> mStats =
+ getMPartitionColumnStatistics(getTable(), partNames, colNames, engine);
+ List<ColumnStatistics> result = new ArrayList<>(Math.min(mStats.size(), partNames.size()));
+ String lastPartName = null;
+ List<ColumnStatisticsObj> curList = null;
+ ColumnStatisticsDesc csd = null;
+ for (int i = 0; i <= mStats.size(); ++i) {
+ boolean isLast = i == mStats.size();
+ MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
+ String partName = isLast ? null : mStatsObj.getPartition().getPartitionName();
+ if (isLast || !partName.equals(lastPartName)) {
+ if (i != 0) {
+ ColumnStatistics colStat = new ColumnStatistics(csd, curList);
+ colStat.setEngine(engine);
+ result.add(colStat);
+ }
+ if (isLast) {
+ continue;
+ }
+ csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
+ curList = new ArrayList<>(colNames.size());
+ }
+ curList.add(StatObjectConverter.getColumnStatisticsObj(mStatsObj, enableBitVector, enableKll));
+ lastPartName = partName;
+ Deadline.checkTimeout();
+ }
+ return result;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames,
+ String engine, String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, return null.
+ if (writeIdList != null) {
+ if (partNames == null || partNames.isEmpty()) {
+ return null;
+ }
+
+ Table table = baseStore.unwrap(TableStore.class).getTable(tableName, null, -1);
+ boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
+ if (isTxn && !areTxnStatsSupported) {
+ return null;
+ }
+ GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+ fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+ fs.setFilters(partNames);
+ GetProjectionsSpec ps = new GetProjectionsSpec();
+ ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%');
+ ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+ List<Partition> parts = baseStore.unwrap(TableStore.class)
+ .getPartitionSpecsByFilterAndProjection(table, ps, fs);
+
+ // Loop through the given "partNames" list
+ // checking isolation-level-compliance of each partition column stats.
+ for (Partition part : parts) {
+
+ if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ LOG.debug("The current metastore transactional partition column "
+ + "statistics for {} {} is not valid for the current query",
+ tableName, partName);
+ return null;
+ }
+ }
+ }
+ return get_aggr_stats_for(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames, String engine)
+ throws MetaException, NoSuchObjectException {
+ final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(conf,
+ MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION);
+ final double ndvTuner = MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER);
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, AggrStats>(this, tableName) {
+ @Override
+ protected AggrStats getSqlResult()
+ throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return getDirectSql().aggrColStatsForPartitions(catName, dbName, tblName, partNames,
+ colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected AggrStats getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using
+ // directSql, do it. No point in failing back to slow path here.
+ throw new MetaException("Jdo path is not implemented for stats aggr.");
+ }
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName)
+ throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(this, null) {
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getSqlResult() throws MetaException {
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using directSql, do it. No point in failing back to slow path
+ // here.
+ throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase.");
+ }
+
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(false);
+ }
+
+ private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(Table table, List<String> partNames,
+ List<String> colNames, String engine) throws MetaException {
+ // We are not going to verify SD for each partition. Just verify for the
+ // table. TODO: we need verify the partition column instead
+ try {
+ validateTableCols(table, colNames);
+ } catch (MetaException me) {
+ LOG.warn("The table does not have the same column definition as its partition.");
+ }
+ List<MPartitionColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4";
+ String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && " +
+ "partition.table.database.catalogName == t3 && engine == t4 && (";
+ Object[] params = new Object[colNames.size() + partNames.size() + 4];
+ int i = 0;
+ params[i++] = table.getTableName();
+ params[i++] = table.getDbName();
+ params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf);
+ params[i++] = engine;
+ int firstI = i;
+ for (String s : partNames) {
+ filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i;
+ paramStr += ", java.lang.String p" + i;
+ params[i++] = s;
+ }
+ filter += ") && (";
+ firstI = i;
+ for (String s : colNames) {
+ filter += ((i == firstI) ? "" : " || ") + "colName == c" + i;
+ paramStr += ", java.lang.String c" + i;
+ params[i++] = s;
+ }
+ filter += ")";
+ query.setFilter(filter);
+ query.declareParameters(paramStr);
+ query.setOrdering("partition.partitionName ascending");
+ result = (List<MPartitionColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(result);
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ String catName = tn.getCat();
+ String dbName = tn.getDb();
+ String tableName = tn.getTable();
+
+ dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME);
+ catName = normalizeIdentifier(catName);
+ if (tableName == null) {
+ throw new RuntimeException("Table name is null.");
+ }
+ MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName);
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4";
+ String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4";
+
+ query.setFilter(filter);
+ query.declareParameters(parameters);
+ query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName),
+ normalizeIdentifier(catName));
+
+ new GetHelper<TableName, Integer>(this, tn) {
+ private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown();
+
+ @Override
+ protected String describeResult() {
+ return "Partition count";
+ }
+
+ @Override
+ protected Integer getSqlResult() throws MetaException {
+ getDirectSql().deleteColumnStatsState(getTable().getId());
+ return 0;
+ }
+
+ @Override
+ protected Integer getJdoResult() throws MetaException, NoSuchObjectException {
+ try {
+ List<Partition> parts = baseStore.unwrap(TableStore.class).getPartitions(tn,
+ GetPartitionsArgs.getAllPartitions());
+ for (Partition part : parts) {
+ Partition newPart = new Partition(part);
+ StatsSetupConst.clearColumnStatsState(newPart.getParameters());
+ baseStore.unwrap(TableStore.class).alterPartition(tn, part.getValues(), newPart, writeIdList);
+ }
+ return parts.size();
+ } catch (InvalidObjectException e) {
+ LOG.error("error updating parts", e);
+ return -1;
+ }
+ }
+ }.run(true);
+ }
+
+ @Override
+ public boolean deletePartitionColumnStatistics(TableName tblName,
+ List<String> partNames, List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ if (partNames == null || partNames.isEmpty()) {
+ throw new InvalidInputException("No partition specified for dropping the statistics");
+ }
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ List<String> cols = normalizeIdentifiers(colNames);
+ return new GetHelper<TableName, Boolean>(this, null) {
+ @Override
+ protected String describeResult() {
+ return "delete partition column stats";
+ }
+ @Override
+ protected Boolean getSqlResult() throws MetaException {
+ DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(getDirectSql(), pm);
+ return deleteStats.deletePartitionColumnStats(catName, dbName, tableName, partNames, cols, engine);
+ }
+ @Override
+ protected Boolean getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ return deletePartitionColumnStatisticsViaJdo(catName, dbName, tableName, partNames, cols, engine);
+ }
+ }.run(false);
+ }
+
+ private boolean deletePartitionColumnStatisticsViaJdo(String catName, String dbName, String tableName,
+ List<String> partNames, List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ boolean ret = false;
Review Comment:
`deletePartitionColumnStatisticsViaJdo` always returns false because `ret` is initialized as false and never updated. On the other hand, `deleteTableColumnStatisticsViaJdo` now always returns true after the work is done. Could we make those two paths consistent? For example, return true after successful JDO work in `deletePartitionColumnStatisticsViaJdo`.
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java:
##########
@@ -0,0 +1,1226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import javax.jdo.Query;
+import javax.jdo.datastore.JDOConnection;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.Batchable;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.Deadline;
+import org.apache.hadoop.hive.metastore.PersistenceManagerProvider;
+import org.apache.hadoop.hive.metastore.QueryWrapper;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionFilterMode;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats;
+import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql;
+import org.apache.hadoop.hive.metastore.metastore.GetHelper;
+import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
+import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
+import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore;
+import org.apache.hadoop.hive.metastore.metastore.iface.TableStore;
+import org.apache.hadoop.hive.metastore.model.MPartition;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MTable;
+import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.utils.RetryingExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING;
+import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams;
+import static org.apache.hadoop.hive.metastore.ObjectStore.isCurrentStatsValidForTheQuery;
+import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers;
+
+public class ColStatsStoreImpl extends RawStoreBundle implements ColStatsStore {
+ private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class);
+
+ private DatabaseProduct dbType;
+ protected int batchSize = NO_BATCHING;
+ private boolean areTxnStatsSupported = false;
+ private Configuration conf;
+ private String schema;
+ private SQLGenerator sqlGenerator;
+
+ @Override
+ public void setBaseStore(RawStore store) {
+ super.setBaseStore(store);
+ this.dbType = PersistenceManagerProvider.getDatabaseProduct();
+ this.batchSize = MetastoreConf.getIntVar(store.getConf(),
+ MetastoreConf.ConfVars.RAWSTORE_PARTITION_BATCH_SIZE);
+ this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(),
+ MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED);
+ this.conf = store.getConf();
+ this.sqlGenerator = new SQLGenerator(dbType, conf);
+ String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema");
+ this.schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null);
+ }
+
+ @Override
+ public List<TableName> getTableNamesWithStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName> (this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getTableNamesWithStats();
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+ }.run(false);
+ }
+
+ @Override
+ public Map<String, List<String>> getPartitionColsWithStats(TableName tableName)
+ throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new GetHelper<TableName, Map<String, List<String>>>(this, null) {
+ @Override
+ protected Map<String, List<String>> getSqlResult() throws MetaException {
+ try {
+ return getDirectSql().getColAndPartNamesWithStats(catName, dbName, tblName);
+ } catch (Throwable ex) {
+ LOG.error("DirectSQL failed", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, List<String>> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+
+ @Override
+ protected String describeResult() {
+ return results.size() + " partitions";
+ }
+ }.run(false);
+ }
+
+ @Override
+ public List<TableName> getAllTableNamesForStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName>(this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getAllTableNamesForStats();
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ List<TableName> result = new ArrayList<>();
+ String paramStr = "", whereStr = "";
+ for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) {
+ if (i != 0) {
+ paramStr += ", ";
+ whereStr += "||";
+ }
+ paramStr += "java.lang.String tt" + i;
+ whereStr += " tableType == tt" + i;
+ }
+ Query query = pm.newQuery(MTable.class, whereStr);
+ query.declareParameters(paramStr);
+ Collection<MTable> tbls =
+ (Collection<MTable>) query.executeWithArray(MetaStoreDirectSql.STATS_TABLE_TYPES);
+ pm.retrieveAll(tbls);
+ for (MTable tbl : tbls) {
+ result.add(new TableName(
+ tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName()));
+ }
+ return result;
+ }
+ }.run(false);
+ }
+
+ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj,
+ MTableColumnStatistics oldStats) throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ String colName = mStatsObj.getColName();
+
+ LOG.info("Updating table level column statistics for table={} colName={}",
+ Warehouse.getCatalogQualifiedTableName(table), colName);
+ validateTableCols(table, Lists.newArrayList(colName));
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ private void writeMPartitionColumnStatistics(Table table, Partition partition,
+ MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) {
+ String catName = mStatsObj.getPartition().getTable().getDatabase().getCatalogName();
+ String dbName = mStatsObj.getPartition().getTable().getDatabase().getName();
+ String tableName = mStatsObj.getPartition().getTable().getTableName();
+ String partName = mStatsObj.getPartition().getPartitionName();
+ String colName = mStatsObj.getColName();
+
+ Preconditions.checkState(this.baseStore.isActiveTransaction());
+
+ LOG.info("Updating partition level column statistics for table=" +
+ TableName.getQualified(catName, dbName, tableName) +
+ " partName=" + partName + " colName=" + colName);
+
+ boolean foundCol = false;
+ List<FieldSchema> colList = partition.getSd().getCols();
+ for (FieldSchema col : colList) {
+ if (col.getName().equals(mStatsObj.getColName())) {
+ foundCol = true;
+ break;
+ }
+ }
+
+ if (!foundCol) {
+ LOG.warn("Column " + colName + " for which stats gathering is requested doesn't exist.");
+ }
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ @Override
+ public Map<String, String> updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ long start = System.currentTimeMillis();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ // DataNucleus objects get detached all over the place for no (real) reason.
+ // So let's not use them anywhere unless absolutely necessary.
+ MTable mTable = baseStore.ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName());
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> transactionHelper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ transactionHelper.setTransactionSavePoint(savePoint);
+ executePlainSQL(
+ sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()),
+ true,
+ exception -> {
+ transactionHelper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mTable);
+ Table table = baseStore.unwrap(TableStore.class).getTable(new TableName(catName, statsDesc.getDbName(),
+ statsDesc.getTableName()), null, -1);
+ List<String> colNames = new ArrayList<>();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+
+ Map<String, MTableColumnStatistics> oldStats = Maps.newHashMap();
+ List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, colStats.getEngine());
+ for (MTableColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc,
+ statsObj, colStats.getEngine());
+ writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName()));
+ // There is no need to add colname again, otherwise we will get duplicate colNames.
+ }
+
+ // Set the table properties
+ // No need to check again if it exists.
+ String dbname = table.getDbName();
+ String name = table.getTableName();
+ MTable oldt = mTable;
+ Map<String, String> newParams = new HashMap<>(table.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters());
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams,
+ writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(oldt.getParameters(), oldt.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name);
+ }
+ oldt.setWriteId(writeId);
+ }
+ }
+ oldt.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+
+ LOG.debug("{} updateTableColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ long start = System.currentTimeMillis();
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ MPartition mPartition =
+ baseStore.ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals);
+ if (mPartition == null) {
+ throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
+ }
+
+ List<String> colNames = new ArrayList<>();
+ for(ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> helper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ helper.setTransactionSavePoint(savePoint);
+ executePlainSQL(sqlGenerator.addForUpdateNoWait(
+ "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()),
+ true,
+ exception -> {
+ helper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mPartition);
+ Partition partition = baseStore.unwrap(TableStore.class)
+ .getPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getValues(), null);
+ Map<String, MPartitionColumnStatistics> oldStats = Maps.newHashMap();
+ List<MPartitionColumnStatistics> stats =
+ getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine());
+ for (MPartitionColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition,
+ statsDesc, statsObj, colStats.getEngine());
+ writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName()));
+ }
+
+ Map<String, String> newParams = new HashMap<>(mPartition.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(table);
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(
+ TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(),
+ newParams, writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName());
+ }
+ mPartition.setWriteId(writeId);
+ }
+ }
+ mPartition.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+ LOG.debug("{} updatePartitionColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
+ Map<String, ColumnStatistics> partColStatsMap,
+ Table tbl,
+ List<TransactionalMetaStoreEventListener> listeners,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+
+ return new GetHelper<TableName, Map<String, Map<String, String>>>(this, null) {
+ @Override
+ protected String describeResult() {
+ return "Map of partition key to column stats if successful";
+ }
+ @Override
+ protected Map<String, Map<String, String>> getSqlResult()
+ throws MetaException {
+ return getDirectSql().updatePartitionColumnStatisticsBatch(partColStatsMap, tbl,
+ listeners, validWriteIds, writeId);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, Map<String, String>> getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled");
+ }
+ }.run(false);
+ }
+
+ private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, String engine)
+ throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ if (colNames.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ validateTableCols(table, colNames);
+
+ List<MTableColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ result =
+ Batchable.runBatched(batchSize, colNames, new Batchable<String, MTableColumnStatistics>() {
+ @Override
+ public List<MTableColumnStatistics> run(List<String> input)
+ throws MetaException {
+ StringBuilder filter =
+ new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && (");
+ StringBuilder paramStr = new StringBuilder(
+ "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4");
+ Object[] params = new Object[input.size() + 4];
+ params[0] = table.getTableName();
+ params[1] = table.getDbName();
+ params[2] = table.getCatName();
+ params[3] = engine;
+ for (int i = 0; i < input.size(); ++i) {
+ filter.append((i == 0) ? "" : " || ").append("colName == c").append(i);
+ paramStr.append(", java.lang.String c").append(i);
+ params[i + 4] = input.get(i);
+ }
+ filter.append(")");
+ query.setFilter(filter.toString());
+ query.declareParameters(paramStr.toString());
+ List<MTableColumnStatistics> paritial = (List<MTableColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(paritial);
+ return paritial;
+ }
+ });
+
+ if (result.size() > colNames.size()) {
+ throw new MetaException("Unexpected " + result.size() + " statistics for "
+ + colNames.size() + " columns");
+ }
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public List<ColumnStatistics> getTableColumnStatistics(
+ TableName tblName,
+ List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<ColumnStatistics> result = new ArrayList<>();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ ColumnStatistics cs = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ return getTableColumnStatisticsInternal(tableName, colNames, engine);
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine,
+ String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, set No to the compliance flag.
+ Boolean isCompliant = null;
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ if (writeIdList != null) {
+ MTable table = baseStore.ensureGetMTable(catName, dbName, tblName);
+ isCompliant = !TxnUtils.isTransactionalTable(table.getParameters())
+ || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table.getParameters(), table.getWriteId(), writeIdList, false));
+ }
+ ColumnStatistics stats = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tblName), colNames, engine);
+ if (stats != null && isCompliant != null) {
+ stats.setIsStatsCompliant(isCompliant);
+ }
+ return stats;
+ }
+
+ protected ColumnStatistics getTableColumnStatisticsInternal(
+ TableName tableName, final List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetStatHelper(tableName, this) {
+ @Override
+ protected ColumnStatistics getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected ColumnStatistics getJdoResult() throws MetaException {
+
+ List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, engine);
+ if (mStats.isEmpty()) {
+ return null;
+ }
+ // LastAnalyzed is stored per column, but thrift object has it per
+ // multiple columns. Luckily, nobody actually uses it, so we will set to
+ // lowest value of all columns for now.
+ ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
+ List<ColumnStatisticsObj> statObjs = new ArrayList<>(mStats.size());
+ for (MTableColumnStatistics mStat : mStats) {
+ if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
+ desc.setLastAnalyzed(mStat.getLastAnalyzed());
+ }
+ statObjs.add(StatObjectConverter.getColumnStatisticsObj(mStat, enableBitVector, enableKll));
+ Deadline.checkTimeout();
+ }
+ ColumnStatistics colStat = new ColumnStatistics(desc, statObjs);
+ colStat.setEngine(engine);
+ return colStat;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<List<ColumnStatistics>> getPartitionColumnStatistics(TableName tblName,
+ List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<List<ColumnStatistics>> result = new ArrayList<>();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ List<ColumnStatistics> cs = getPartitionColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), partNames, colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(TableName tableName,
+ List<String> partNames, List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ return getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ TableName tableName,
+ List<String> partNames, List<String> colNames,
+ String engine, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ List<ColumnStatistics> allStats = getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ if (writeIdList != null) {
+ if (!areTxnStatsSupported) {
+ for (ColumnStatistics cs : allStats) {
+ cs.setIsStatsCompliant(false);
+ }
+ } else {
+ // TODO: this could be improved to get partitions in bulk
+ for (ColumnStatistics cs : allStats) {
+ MPartition mpart = baseStore.ensureGetMPartition(tableName,
+ Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName()));
+ if (mpart == null
+ || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) {
+ if (mpart != null) {
+ LOG.debug("The current metastore transactional partition column statistics for {} {} "
+ + "(write ID {}) are not valid for current query ({})", tableName,
+ mpart.getPartitionName(), mpart.getWriteId(), writeIdList);
+ }
+ cs.setIsStatsCompliant(false);
+ } else {
+ cs.setIsStatsCompliant(true);
+ }
+ }
+ }
+ }
+ return allStats;
+ }
+
+ protected List<ColumnStatistics> getPartitionColumnStatisticsInternal(
+ TableName tableName, final List<String> partNames, final List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetListHelper<TableName, ColumnStatistics>(this, tableName) {
+ @Override
+ protected List<ColumnStatistics> getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema).getPartitionStats(
+ catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll);
+ }
+ @Override
+ protected List<ColumnStatistics> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ List<MPartitionColumnStatistics> mStats =
+ getMPartitionColumnStatistics(getTable(), partNames, colNames, engine);
+ List<ColumnStatistics> result = new ArrayList<>(Math.min(mStats.size(), partNames.size()));
+ String lastPartName = null;
+ List<ColumnStatisticsObj> curList = null;
+ ColumnStatisticsDesc csd = null;
+ for (int i = 0; i <= mStats.size(); ++i) {
+ boolean isLast = i == mStats.size();
+ MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
+ String partName = isLast ? null : mStatsObj.getPartition().getPartitionName();
+ if (isLast || !partName.equals(lastPartName)) {
+ if (i != 0) {
+ ColumnStatistics colStat = new ColumnStatistics(csd, curList);
+ colStat.setEngine(engine);
+ result.add(colStat);
+ }
+ if (isLast) {
+ continue;
+ }
+ csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
+ curList = new ArrayList<>(colNames.size());
+ }
+ curList.add(StatObjectConverter.getColumnStatisticsObj(mStatsObj, enableBitVector, enableKll));
+ lastPartName = partName;
+ Deadline.checkTimeout();
+ }
+ return result;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames,
+ String engine, String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, return null.
+ if (writeIdList != null) {
+ if (partNames == null || partNames.isEmpty()) {
+ return null;
+ }
+
+ Table table = baseStore.unwrap(TableStore.class).getTable(tableName, null, -1);
+ boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
+ if (isTxn && !areTxnStatsSupported) {
+ return null;
+ }
+ GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+ fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+ fs.setFilters(partNames);
+ GetProjectionsSpec ps = new GetProjectionsSpec();
+ ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%');
+ ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+ List<Partition> parts = baseStore.unwrap(TableStore.class)
+ .getPartitionSpecsByFilterAndProjection(table, ps, fs);
+
+ // Loop through the given "partNames" list
+ // checking isolation-level-compliance of each partition column stats.
+ for (Partition part : parts) {
+
+ if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ LOG.debug("The current metastore transactional partition column "
+ + "statistics for {} {} is not valid for the current query",
+ tableName, partName);
+ return null;
+ }
+ }
+ }
+ return get_aggr_stats_for(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames, String engine)
+ throws MetaException, NoSuchObjectException {
+ final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(conf,
+ MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION);
+ final double ndvTuner = MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER);
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, AggrStats>(this, tableName) {
+ @Override
+ protected AggrStats getSqlResult()
+ throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return getDirectSql().aggrColStatsForPartitions(catName, dbName, tblName, partNames,
+ colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected AggrStats getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using
+ // directSql, do it. No point in failing back to slow path here.
+ throw new MetaException("Jdo path is not implemented for stats aggr.");
+ }
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName)
+ throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(this, null) {
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getSqlResult() throws MetaException {
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using directSql, do it. No point in failing back to slow path
+ // here.
+ throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase.");
+ }
+
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(false);
+ }
+
+ private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(Table table, List<String> partNames,
+ List<String> colNames, String engine) throws MetaException {
+ // We are not going to verify SD for each partition. Just verify for the
+ // table. TODO: we need verify the partition column instead
+ try {
+ validateTableCols(table, colNames);
+ } catch (MetaException me) {
+ LOG.warn("The table does not have the same column definition as its partition.");
+ }
+ List<MPartitionColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4";
+ String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && " +
+ "partition.table.database.catalogName == t3 && engine == t4 && (";
+ Object[] params = new Object[colNames.size() + partNames.size() + 4];
+ int i = 0;
+ params[i++] = table.getTableName();
+ params[i++] = table.getDbName();
+ params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf);
+ params[i++] = engine;
+ int firstI = i;
+ for (String s : partNames) {
+ filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i;
+ paramStr += ", java.lang.String p" + i;
+ params[i++] = s;
+ }
+ filter += ") && (";
+ firstI = i;
+ for (String s : colNames) {
+ filter += ((i == firstI) ? "" : " || ") + "colName == c" + i;
+ paramStr += ", java.lang.String c" + i;
+ params[i++] = s;
+ }
+ filter += ")";
+ query.setFilter(filter);
+ query.declareParameters(paramStr);
+ query.setOrdering("partition.partitionName ascending");
+ result = (List<MPartitionColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(result);
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ String catName = tn.getCat();
+ String dbName = tn.getDb();
+ String tableName = tn.getTable();
+
+ dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME);
+ catName = normalizeIdentifier(catName);
+ if (tableName == null) {
+ throw new RuntimeException("Table name is null.");
+ }
+ MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName);
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4";
+ String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4";
+
+ query.setFilter(filter);
+ query.declareParameters(parameters);
+ query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName),
+ normalizeIdentifier(catName));
+
+ new GetHelper<TableName, Integer>(this, tn) {
+ private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown();
+
+ @Override
+ protected String describeResult() {
+ return "Partition count";
+ }
+
+ @Override
+ protected Integer getSqlResult() throws MetaException {
+ getDirectSql().deleteColumnStatsState(getTable().getId());
+ return 0;
+ }
+
+ @Override
+ protected Integer getJdoResult() throws MetaException, NoSuchObjectException {
+ try {
+ List<Partition> parts = baseStore.unwrap(TableStore.class).getPartitions(tn,
+ GetPartitionsArgs.getAllPartitions());
+ for (Partition part : parts) {
+ Partition newPart = new Partition(part);
+ StatsSetupConst.clearColumnStatsState(newPart.getParameters());
+ baseStore.unwrap(TableStore.class).alterPartition(tn, part.getValues(), newPart, writeIdList);
+ }
+ return parts.size();
+ } catch (InvalidObjectException e) {
+ LOG.error("error updating parts", e);
+ return -1;
+ }
+ }
+ }.run(true);
+ }
+
+ @Override
+ public boolean deletePartitionColumnStatistics(TableName tblName,
+ List<String> partNames, List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ if (partNames == null || partNames.isEmpty()) {
+ throw new InvalidInputException("No partition specified for dropping the statistics");
+ }
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ List<String> cols = normalizeIdentifiers(colNames);
+ return new GetHelper<TableName, Boolean>(this, null) {
+ @Override
+ protected String describeResult() {
+ return "delete partition column stats";
+ }
+ @Override
+ protected Boolean getSqlResult() throws MetaException {
+ DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(getDirectSql(), pm);
+ return deleteStats.deletePartitionColumnStats(catName, dbName, tableName, partNames, cols, engine);
+ }
+ @Override
+ protected Boolean getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ return deletePartitionColumnStatisticsViaJdo(catName, dbName, tableName, partNames, cols, engine);
+ }
+ }.run(false);
+ }
+
+ private boolean deletePartitionColumnStatisticsViaJdo(String catName, String dbName, String tableName,
+ List<String> partNames, List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ boolean ret = false;
+ String database = org.apache.commons.lang3.StringUtils.defaultString(dbName,
+ Warehouse.DEFAULT_DATABASE_NAME);
+ String catalog = normalizeIdentifier(catName);
+ Batchable<String, Void> b = new Batchable<String, Void>() {
+ @Override
+ public List<Void> run(List<String> input) throws Exception {
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ addQueryAfterUse(query);
+ String filter;
+ String parameters;
+ if (colNames != null && !colNames.isEmpty()) {
+ filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && "
+ + "t4.contains(colName) && partition.table.database.catalogName == t5" + (engine != null ? " && engine == t6" : "");
+ parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, "
+ + "java.util.Collection t4, java.lang.String t5" + (engine != null ? ", java.lang.String t6" : "");
+ } else {
+ filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " +
+ "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : "");
+ parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"
+ + (engine != null ? ", java.lang.String t5" : "");
+ }
+ query.setFilter(filter);
+ query.declareParameters(parameters);
+ List<Object> params = new ArrayList<>();
+ params.add(input);
+ params.add(normalizeIdentifier(database));
+ params.add(normalizeIdentifier(tableName));
+ if (colNames != null && !colNames.isEmpty()) {
+ params.add(colNames);
+ }
+ params.add(catalog);
+ if (engine != null) {
+ params.add(engine);
+ }
+ List<MPartitionColumnStatistics> mStatsObjColl =
+ (List<MPartitionColumnStatistics>) query.executeWithArray(params.toArray());
+ pm.retrieveAll(mStatsObjColl);
+ if (mStatsObjColl != null) {
+ pm.deletePersistentAll(mStatsObjColl);
+ }
+ return null;
+ }
+ };
+ try {
+ Batchable.runBatched(batchSize, partNames, b);
+ } finally {
+ b.closeAllQueries();
+ }
+
+ Batchable.runBatched(batchSize, partNames, new Batchable<String, Void>() {
+ @Override
+ public List<Void> run(List<String> input) throws MetaException {
+ Pair<Query, Map<String, String>> queryWithParams =
+ getPartQueryWithParams(pm, catalog, database, tableName, input);
+ try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) {
+ qw.setResultClass(MPartition.class);
+ qw.setClass(MPartition.class);
+ List<MPartition> mparts = (List<MPartition>) qw.executeWithMap(queryWithParams.getRight());
+ for (MPartition mPart : mparts) {
+ Map<String, String> params = mPart.getParameters();
+ if (params != null && params.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) {
+ if (colNames == null || colNames.isEmpty()) {
+ StatsSetupConst.clearColumnStatsState(params);
+ } else {
+ StatsSetupConst.removeColumnStatsState(params, colNames);
+ }
+ mPart.setParameters(params);
+ }
+ }
+ }
+ return Collections.emptyList();
+ }
+ });
+ return ret;
+ }
+
+ @Override
+ public boolean deleteTableColumnStatistics(TableName tableName,
+ List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ if (tableName == null) {
+ throw new InvalidInputException("Table name is null.");
+ }
+ List<String> cols = normalizeIdentifiers(colNames);
+ return new GetHelper<TableName, Boolean>(this, tableName) {
+ @Override
+ protected String describeResult() {
+ return "delete table column stats";
+ }
+ @Override
+ protected Boolean getSqlResult() throws MetaException {
+ DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(getDirectSql(), pm);
+ return deleteStats.deleteTableColumnStatistics(getTable(), cols, engine);
+ }
+ @Override
+ protected Boolean getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ return deleteTableColumnStatisticsViaJdo(tableName, cols, engine);
+ }
+ }.run(true);
+ }
+
+ private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, List<String> colNames, String engine)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ List<MTableColumnStatistics> mStatsObjColl;
+ // Note: this does not verify ACID state; called internally when removing cols/etc.
+ // Also called via an unused metastore API that checks for ACID tables.
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ String filter;
+ String parameters;
+ if (colNames != null && !colNames.isEmpty()) {
+ filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)"
+ + (engine != null ? " && engine == t5" : "");
+ parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4"
+ + (engine != null ? ", java.lang.String t5" : "");
+ } else {
+ filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"
+ + (engine != null ? " && engine == t4" : "");
+ parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3"
+ + (engine != null ? ", java.lang.String t4" : "");
+ }
+
+ query.setFilter(filter);
+ query.declareParameters(parameters);
+ List<Object> params = new ArrayList<>();
+ params.add(normalizeIdentifier(tableName));
+ params.add(normalizeIdentifier(dbName));
+ params.add(catName == null ? null : normalizeIdentifier(catName));
+ if (colNames != null && !colNames.isEmpty()) {
+ params.add(colNames);
+ }
+ if (engine != null) {
+ params.add(engine);
+ }
+ mStatsObjColl = (List<MTableColumnStatistics>) query.executeWithArray(params.toArray());
+ pm.retrieveAll(mStatsObjColl);
+ if (mStatsObjColl != null) {
+ pm.deletePersistentAll(mStatsObjColl);
+ }
+
+ MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName);
+ if (mTable != null) {
+ Map<String, String> tableParams = mTable.getParameters();
+ if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) {
+ if (colNames == null || colNames.isEmpty()) {
+ StatsSetupConst.clearColumnStatsState(tableParams);
+ } else {
+ StatsSetupConst.removeColumnStatsState(tableParams, colNames);
+ }
+ mTable.setParameters(tableParams);
+ }
+ }
+ return true;
+ }
+
+ private void executePlainSQL(String sql,
+ boolean atLeastOneRecord,
+ Consumer<Exception> exceptionConsumer)
+ throws SQLException, MetaException {
+ String s = dbType.getPrepareTxnStmt();
+ assert pm.currentTransaction().isActive();
+ JDOConnection jdoConn = pm.getDataStoreConnection();
+ Connection conn = (Connection) jdoConn.getNativeConnection();
+ try (Statement statement = conn.createStatement()) {
+ if (s != null) {
+ statement.execute(s);
+ }
+ try {
+ statement.execute(sql);
+ try (ResultSet rs = statement.getResultSet()) {
+ // sqlserver needs rs.next for validating the s4u nowait
+ if (atLeastOneRecord && !rs.next()) {
+ throw new MetaException("At least one record but none is returned from the query: " + sql);
+ }
+ }
+ } catch (SQLException e) {
+ if (exceptionConsumer != null) {
+ exceptionConsumer.accept(e);
+ } else {
+ throw e;
+ }
+ }
+ } finally {
+ jdoConn.close();
+ }
+ }
+
+ private abstract class GetStatHelper extends GetHelper<TableName, ColumnStatistics> {
Review Comment:
nit, GetStatHelper is used only once. Could we replace it with GetHelper<TableName, ColumnStatistics> like other methods?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java:
##########
@@ -0,0 +1,1053 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore.impl;
+
+import com.google.common.collect.Sets;
+
+import javax.jdo.Query;
+import java.sql.SQLIntegrityConstraintViolationException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.QueryWrapper;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.RuntimeStat;
+import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMMapping;
+import org.apache.hadoop.hive.metastore.api.WMNullablePool;
+import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMPool;
+import org.apache.hadoop.hive.metastore.api.WMPoolTrigger;
+import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus;
+import org.apache.hadoop.hive.metastore.api.WMTrigger;
+import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
+import org.apache.hadoop.hive.metastore.metastore.iface.WMStore;
+import org.apache.hadoop.hive.metastore.model.MRuntimeStat;
+import org.apache.hadoop.hive.metastore.model.MWMMapping;
+import org.apache.hadoop.hive.metastore.model.MWMPool;
+import org.apache.hadoop.hive.metastore.model.MWMResourcePlan;
+import org.apache.hadoop.hive.metastore.model.MWMTrigger;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+public class WMStoreImpl extends RawStoreBundle implements WMStore {
+ private static final Logger LOG = LoggerFactory.getLogger(WMStoreImpl.class);
+ private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern(
+ "yyyy_MM_dd_HH_mm_ss");
+
+ private void checkForConstraintException(Exception e, String msg) throws AlreadyExistsException {
+ if (getConstraintException(e) != null) {
+ LOG.error(msg, e);
+ throw new AlreadyExistsException(msg);
+ }
+ }
+
+ private Throwable getConstraintException(Throwable t) {
+ while (t != null) {
+ if (t instanceof SQLIntegrityConstraintViolationException) {
+ return t;
+ }
+ t = t.getCause();
+ }
+ return null;
+ }
+
+ @Override
+ public void createResourcePlan(
+ WMResourcePlan resourcePlan, String copyFromName, int defaultPoolSize)
+ throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException {
+ String rpName = normalizeIdentifier(resourcePlan.getName());
+ if (rpName.isEmpty()) {
+ throw new InvalidObjectException("Resource name cannot be empty.");
+ }
+ MWMResourcePlan rp = null;
+ if (copyFromName == null) {
+ Integer queryParallelism = null;
+ if (resourcePlan.isSetQueryParallelism()) {
+ queryParallelism = resourcePlan.getQueryParallelism();
+ if (queryParallelism <= 0) {
+ throw new InvalidObjectException("Query parallelism should be positive.");
+ }
+ }
+ rp = new MWMResourcePlan(rpName, queryParallelism, MWMResourcePlan.Status.DISABLED);
+ } else {
+ rp = new MWMResourcePlan(rpName, null, MWMResourcePlan.Status.DISABLED);
+ }
+ rp.setNs(resourcePlan.getNs());
+ try {
+ pm.makePersistent(rp);
+ if (copyFromName != null) {
+ String ns = getNsOrDefault(resourcePlan.getNs());
+ MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, ns, false);
+ if (copyFrom == null) {
+ throw new NoSuchObjectException(copyFromName + " in " + ns);
+ }
+ copyRpContents(rp, copyFrom);
+ } else {
+ // TODO: ideally, this should be moved outside to HiveMetaStore to be shared between
+ // all the RawStore-s. Right now there's no method to create a pool.
+ if (defaultPoolSize > 0) {
+ MWMPool defaultPool = new MWMPool(rp, "default", 1.0, defaultPoolSize, null);
+ pm.makePersistent(defaultPool);
+ rp.setPools(Sets.newHashSet(defaultPool));
+ rp.setDefaultPool(defaultPool);
+ }
+ }
+ } catch (InvalidOperationException e) {
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ checkForConstraintException(e, "Resource plan already exists: " + rpName);
+ throw e;
+ }
+ }
+
+ private void copyRpContents(MWMResourcePlan dest, MWMResourcePlan src) {
+ dest.setQueryParallelism(src.getQueryParallelism());
+ dest.setNs(src.getNs());
+ Map<String, MWMPool> pools = new HashMap<>();
+ Map<String, Set<MWMPool>> triggersToPools = new HashMap<>();
+ for (MWMPool copyPool : src.getPools()) {
+ MWMPool pool = new MWMPool(dest, copyPool.getPath(), copyPool.getAllocFraction(),
+ copyPool.getQueryParallelism(), copyPool.getSchedulingPolicy());
+ pm.makePersistent(pool);
+ pools.put(copyPool.getPath(), pool);
+ if (copyPool.getTriggers() != null) {
+ for (MWMTrigger trigger : copyPool.getTriggers()) {
+ Set<MWMPool> p2t = triggersToPools.get(trigger.getName());
+ if (p2t == null) {
+ p2t = new HashSet<>();
+ triggersToPools.put(trigger.getName(), p2t);
+ }
+ p2t.add(pool);
+ pool.setTriggers(new HashSet<>());
+ }
+ }
+ }
+ dest.setPools(new HashSet<>(pools.values()));
+ if (src.getDefaultPool() != null) {
+ dest.setDefaultPool(pools.get(src.getDefaultPool().getPath()));
+ }
+ Set<MWMMapping> mappings = new HashSet<>();
+ for (MWMMapping copyMapping : src.getMappings()) {
+ MWMPool pool = null;
+ if (copyMapping.getPool() != null) {
+ pool = pools.get(copyMapping.getPool().getPath());
+ }
+ MWMMapping mapping = new MWMMapping(dest, copyMapping.getEntityType(),
+ copyMapping.getEntityName(), pool, copyMapping.getOrdering());
+ pm.makePersistent(mapping);
+ mappings.add(mapping);
+ }
+ dest.setMappings(mappings);
+ Set<MWMTrigger> triggers = new HashSet<>();
+ for (MWMTrigger copyTrigger : src.getTriggers()) {
+ Set<MWMPool> p2t = triggersToPools.get(copyTrigger.getName());
+ if (p2t == null) {
+ p2t = new HashSet<>();
+ }
+ MWMTrigger trigger = new MWMTrigger(dest, copyTrigger.getName(),
+ copyTrigger.getTriggerExpression(), copyTrigger.getActionExpression(), p2t,
+ copyTrigger.getIsInUnmanaged());
+ pm.makePersistent(trigger);
+ for (MWMPool pool : p2t) {
+ pool.getTriggers().add(trigger);
+ }
+ triggers.add(trigger);
+ }
+ dest.setTriggers(triggers);
+ }
+
+ private WMResourcePlan fromMResourcePlan(MWMResourcePlan mplan) {
+ if (mplan == null) {
+ return null;
+ }
+ WMResourcePlan rp = new WMResourcePlan();
+ rp.setName(mplan.getName());
+ rp.setNs(mplan.getNs());
+ rp.setStatus(WMResourcePlanStatus.valueOf(mplan.getStatus().name()));
+ if (mplan.getQueryParallelism() != null) {
+ rp.setQueryParallelism(mplan.getQueryParallelism());
+ }
+ if (mplan.getDefaultPool() != null) {
+ rp.setDefaultPoolPath(mplan.getDefaultPool().getPath());
+ }
+ return rp;
+ }
+
+ private WMFullResourcePlan fullFromMResourcePlan(MWMResourcePlan mplan) {
+ if (mplan == null) {
+ return null;
+ }
+ WMFullResourcePlan rp = new WMFullResourcePlan();
+ rp.setPlan(fromMResourcePlan(mplan));
+ for (MWMPool mPool : mplan.getPools()) {
+ rp.addToPools(fromMPool(mPool, mplan.getName()));
+ for (MWMTrigger mTrigger : mPool.getTriggers()) {
+ rp.addToPoolTriggers(new WMPoolTrigger(mPool.getPath(), mTrigger.getName()));
+ }
+ }
+ for (MWMTrigger mTrigger : mplan.getTriggers()) {
+ rp.addToTriggers(fromMWMTrigger(mTrigger, mplan.getName()));
+ }
+ for (MWMMapping mMapping : mplan.getMappings()) {
+ rp.addToMappings(fromMMapping(mMapping, mplan.getName()));
+ }
+ return rp;
+ }
+
+ private WMPool fromMPool(MWMPool mPool, String rpName) {
+ WMPool result = new WMPool(rpName, mPool.getPath());
+ assert mPool.getAllocFraction() != null;
+ result.setAllocFraction(mPool.getAllocFraction());
+ assert mPool.getQueryParallelism() != null;
+ result.setQueryParallelism(mPool.getQueryParallelism());
+ result.setSchedulingPolicy(mPool.getSchedulingPolicy());
+ result.setNs(mPool.getResourcePlan().getNs());
+ return result;
+ }
+
+ private WMMapping fromMMapping(MWMMapping mMapping, String rpName) {
+ WMMapping result = new WMMapping(rpName,
+ mMapping.getEntityType().toString(), mMapping.getEntityName());
+ if (mMapping.getPool() != null) {
+ result.setPoolPath(mMapping.getPool().getPath());
+ }
+ if (mMapping.getOrdering() != null) {
+ result.setOrdering(mMapping.getOrdering());
+ }
+ result.setNs(mMapping.getResourcePlan().getNs());
+ return result;
+ }
+
+ private String getNsOrDefault(String ns) {
+ // This is only needed for old clients not setting NS in requests.
+ // Not clear how to handle this... this is properly a HS2 config but metastore needs its default
+ // value for backward compat, and we don't want it configurable separately because it's also
+ // used in upgrade scripts, were it cannot be configured.
+ return normalizeIdentifier(ns == null ? "default" : ns);
+ }
+
+ @Override
+ public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException {
+ try {
+ return fullFromMResourcePlan(getMWMResourcePlan(name, ns, false));
+ } catch (InvalidOperationException e) {
+ // Should not happen, edit check is false.
+ throw new RuntimeException(e);
+ }
+ }
+
+ private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck)
+ throws NoSuchObjectException, InvalidOperationException {
+ return getMWMResourcePlan(name, ns, editCheck, true);
+ }
+
+ private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck, boolean mustExist)
+ throws NoSuchObjectException, InvalidOperationException {
+ MWMResourcePlan resourcePlan;
+
+ name = normalizeIdentifier(name);
+ Query query = createGetResourcePlanQuery();
+ ns = getNsOrDefault(ns);
+ resourcePlan = (MWMResourcePlan) query.execute(name, ns);
+ if (resourcePlan != null) {
+ pm.retrieve(resourcePlan);
+ }
+ if (mustExist && resourcePlan == null) {
+ throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns);
+ }
+ if (editCheck && resourcePlan != null
+ && resourcePlan.getStatus() != MWMResourcePlan.Status.DISABLED) {
+ throw new InvalidOperationException("Resource plan must be disabled to edit it.");
+ }
+ return resourcePlan;
+ }
+
+ private Query createGetResourcePlanQuery() {
+ Query query = pm.newQuery(MWMResourcePlan.class, "name == rpname && ns == nsname");
+ query.declareParameters("java.lang.String rpname, java.lang.String nsname");
+ query.setUnique(true);
+ return query;
+ }
+
+ @Override
+ public List<WMResourcePlan> getAllResourcePlans(String ns) throws MetaException {
+ List<WMResourcePlan> resourcePlans = new ArrayList();
+ Query query = pm.newQuery(MWMResourcePlan.class, "ns == nsname");
+ query.declareParameters("java.lang.String nsname");
+ List<MWMResourcePlan> mplans = (List<MWMResourcePlan>) query.execute(getNsOrDefault(ns));
+ pm.retrieveAll(mplans);
+ if (mplans != null) {
+ for (MWMResourcePlan mplan : mplans) {
+ resourcePlans.add(fromMResourcePlan(mplan));
+ }
+ }
+ return resourcePlans;
+ }
+
+ @Override
+ public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan changes,
+ boolean canActivateDisabled, boolean canDeactivate, boolean isReplace)
+ throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException {
+ name = name == null ? null : normalizeIdentifier(name);
+ if (isReplace && name == null) {
+ throw new InvalidOperationException("Cannot replace without specifying the source plan");
+ }
+ // This method only returns the result when activating a resource plan.
+ // We could also add a boolean flag to be specified by the caller to see
+ // when the result might be needed.
+ WMFullResourcePlan result = null;
+ try {
+ if (isReplace) {
+ result = handleAlterReplace(name, ns, changes);
+ } else {
+ result = handleSimpleAlter(name, ns, changes, canActivateDisabled, canDeactivate);
+ }
+ return result;
+ } catch (Exception e) {
+ checkForConstraintException(e, "Resource plan name should be unique: ");
+ throw e;
+ }
+ }
+
+ private WMFullResourcePlan handleSimpleAlter(String name, String ns, WMNullableResourcePlan changes,
+ boolean canActivateDisabled, boolean canDeactivate)
+ throws InvalidOperationException, NoSuchObjectException, MetaException {
+ MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan(ns)
+ : getMWMResourcePlan(name, ns, !changes.isSetStatus());
+ boolean hasNsChange = changes.isSetNs() && !changes.getNs().equals(getNsOrDefault(plan.getNs()));
+ if (hasNsChange) {
+ throw new InvalidOperationException("Cannot change ns; from " + getNsOrDefault(plan.getNs())
+ + " to " + changes.getNs());
+ }
+ boolean hasNameChange = changes.isSetName() && !changes.getName().equals(name);
+ // Verify that field changes are consistent with what Hive does. Note: we could handle this.
+ if (changes.isSetIsSetQueryParallelism()
+ || changes.isSetIsSetDefaultPoolPath() || hasNameChange) {
+ if (changes.isSetStatus()) {
+ throw new InvalidOperationException("Cannot change values during status switch.");
+ } else if (plan.getStatus() != MWMResourcePlan.Status.DISABLED) {
+ throw new InvalidOperationException("Resource plan must be disabled to edit it.");
+ }
+ }
+
+ // Handle rename and other changes.
+ if (changes.isSetName()) {
+ String newName = normalizeIdentifier(changes.getName());
+ if (newName.isEmpty()) {
+ throw new InvalidOperationException("Cannot rename to empty value.");
+ }
+ if (!newName.equals(plan.getName())) {
+ plan.setName(newName);
+ }
+ }
+ if (changes.isSetIsSetQueryParallelism() && changes.isIsSetQueryParallelism()) {
+ if (changes.isSetQueryParallelism()) {
+ if (changes.getQueryParallelism() <= 0) {
+ throw new InvalidOperationException("queryParallelism should be positive.");
+ }
+ plan.setQueryParallelism(changes.getQueryParallelism());
+ } else {
+ plan.setQueryParallelism(null);
+ }
+ }
+ if (changes.isSetIsSetDefaultPoolPath() && changes.isIsSetDefaultPoolPath()) {
+ if (changes.isSetDefaultPoolPath()) {
+ MWMPool pool = getPool(plan, changes.getDefaultPoolPath());
+ plan.setDefaultPool(pool);
+ } else {
+ plan.setDefaultPool(null);
+ }
+ }
+
+ // Handle the status change.
+ if (changes.isSetStatus()) {
+ return switchStatus(name, plan,
+ changes.getStatus().name(), canActivateDisabled, canDeactivate);
+ }
+ return null;
+ }
+
+ private WMFullResourcePlan handleAlterReplace(String name, String ns, WMNullableResourcePlan changes)
+ throws InvalidOperationException, NoSuchObjectException, MetaException {
+ // Verify that field changes are consistent with what Hive does. Note: we could handle this.
+ if (changes.isSetQueryParallelism() || changes.isSetDefaultPoolPath()) {
+ throw new InvalidOperationException("Cannot change values during replace.");
+ }
+ boolean isReplacingSpecific = changes.isSetName();
+ boolean isReplacingActive = (changes.isSetStatus()
+ && changes.getStatus() == WMResourcePlanStatus.ACTIVE);
+ if (isReplacingActive == isReplacingSpecific) {
+ throw new InvalidOperationException("Must specify a name, or the active plan; received "
+ + changes.getName() + ", " + (changes.isSetStatus() ? changes.getStatus() : null));
+ }
+ if (name == null) {
+ throw new InvalidOperationException("Invalid replace - no name specified");
+ }
+ ns = getNsOrDefault(ns);
+ MWMResourcePlan replacedPlan = isReplacingSpecific
+ ? getMWMResourcePlan(changes.getName(), ns, false) : getActiveMWMResourcePlan(ns);
+ MWMResourcePlan plan = getMWMResourcePlan(name, ns, false);
+
+ if (replacedPlan.getName().equals(plan.getName())) {
+ throw new InvalidOperationException("A plan cannot replace itself");
+ }
+ String oldNs = getNsOrDefault(replacedPlan.getNs()), newNs = getNsOrDefault(plan.getNs());
+ if (!oldNs.equals(newNs)) {
+ throw new InvalidOperationException("Cannot change the namespace; replacing "
+ + oldNs + " with " + newNs);
+ }
+
+ // We will inherit the name and status from the plan we are replacing.
+ String newName = replacedPlan.getName();
+ int i = 0;
+ String copyName = generateOldPlanName(newName, i);
+ while (true) {
+ MWMResourcePlan dup = getMWMResourcePlan(copyName, ns, false, false);
+ if (dup == null) {
+ break;
+ }
+ // Note: this can still conflict with parallel transactions. We do not currently handle
+ // parallel changes from two admins (by design :().
+ copyName = generateOldPlanName(newName, ++i);
+ }
+ replacedPlan.setName(copyName);
+ plan.setName(newName);
+ plan.setStatus(replacedPlan.getStatus());
+ replacedPlan.setStatus(MWMResourcePlan.Status.DISABLED);
+ // TODO: add a configurable option to skip the history and just drop it?
+ return plan.getStatus() == MWMResourcePlan.Status.ACTIVE ? fullFromMResourcePlan(plan) : null;
+ }
+
+ private String generateOldPlanName(String newName, int i) {
+ if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) {
+ // Do not use datetime in tests to avoid result changes.
+ return newName + "_old_" + i;
+ } else {
+ return newName + "_old_"
+ + LocalDateTime.now().format(YMDHMS_FORMAT) + (i == 0 ? "" : ("_" + i));
+ }
+ }
+
+ @Override
+ public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException {
+ // Note: fullFromMResroucePlan needs to be called inside the txn, otherwise we could have
+ // deduplicated this with getActiveMWMResourcePlan.
+ WMFullResourcePlan result = null;
+ Query query = createActivePlanQuery();
+ MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(
+ MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns));
+ if (mResourcePlan != null) {
+ result = fullFromMResourcePlan(mResourcePlan);
+ }
+ return result;
+ }
+
+ private MWMResourcePlan getActiveMWMResourcePlan(String ns) {
+ MWMResourcePlan result = null;
+ Query query = createActivePlanQuery();
+ result = (MWMResourcePlan) query.execute(
+ MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns));
+ if (result != null) {
+ pm.retrieve(result);
+ }
+ return result;
+ }
+
+ private Query createActivePlanQuery() {
+ Query query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus && ns == nsname");
+ query.declareParameters("java.lang.String activeStatus, java.lang.String nsname");
+ query.setUnique(true);
+ return query;
+ }
+
+ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePlan, String status,
+ boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException {
+ MWMResourcePlan.Status currentStatus = mResourcePlan.getStatus();
+ MWMResourcePlan.Status newStatus = null;
+ try {
+ newStatus = MWMResourcePlan.Status.valueOf(status);
+ } catch (IllegalArgumentException e) {
+ throw new InvalidOperationException("Invalid status: " + status);
+ }
+
+ if (newStatus == currentStatus) {
+ return null;
+ }
+
+ boolean doActivate = false, doValidate = false;
+ switch (currentStatus) {
+ case ACTIVE: // No status change for active resource plan, first activate another plan.
+ if (!canDeactivate) {
+ throw new InvalidOperationException(
+ "Resource plan " + name
+ + " is active; activate another plan first, or disable workload management.");
+ }
+ break;
+ case DISABLED:
+ assert newStatus == MWMResourcePlan.Status.ACTIVE || newStatus == MWMResourcePlan.Status.ENABLED;
+ doValidate = true;
+ doActivate = (newStatus == MWMResourcePlan.Status.ACTIVE);
+ if (doActivate && !canActivateDisabled) {
+ throw new InvalidOperationException("Resource plan " + name
+ + " is disabled and should be enabled before activation (or in the same command)");
+ }
+ break;
+ case ENABLED:
+ if (newStatus == MWMResourcePlan.Status.DISABLED) {
+ mResourcePlan.setStatus(newStatus);
+ return null; // A simple case.
+ }
+ assert newStatus == MWMResourcePlan.Status.ACTIVE;
+ doActivate = true;
+ break;
+ default: throw new AssertionError("Unexpected status " + currentStatus);
+ }
+ if (doValidate) {
+ // Note: this may use additional inputs from the caller, e.g. maximum query
+ // parallelism in the cluster based on physical constraints.
+ WMValidateResourcePlanResponse response = getResourcePlanErrors(mResourcePlan);
+ if (!response.getErrors().isEmpty()) {
+ throw new InvalidOperationException(
+ "ResourcePlan: " + name + " is invalid: " + response.getErrors());
+ }
+ }
+ if (doActivate) {
+ // Deactivate currently active resource plan.
+ deactivateActiveResourcePlan(mResourcePlan.getNs());
+ mResourcePlan.setStatus(newStatus);
+ return fullFromMResourcePlan(mResourcePlan);
+ } else {
+ mResourcePlan.setStatus(newStatus);
+ }
+ return null;
+ }
+
+ private void deactivateActiveResourcePlan(String ns) {
+ Query query = createActivePlanQuery();
+ MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(
+ MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns));
+ // We may not have an active resource plan in the start.
+ if (mResourcePlan != null) {
+ mResourcePlan.setStatus(MWMResourcePlan.Status.ENABLED);
+ }
+ }
+
+ private static class PoolData {
+ double totalChildrenAllocFraction = 0;
+ boolean found = false;
+ boolean hasChildren = false;
+ }
+
+ private PoolData getPoolData(Map<String, PoolData> poolInfo, String poolPath) {
+ PoolData poolData = poolInfo.get(poolPath);
+ if (poolData == null) {
+ poolData = new PoolData();
+ poolInfo.put(poolPath, poolData);
+ }
+ return poolData;
+ }
+
+ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mResourcePlan) {
+ WMValidateResourcePlanResponse response = new WMValidateResourcePlanResponse();
+ response.setErrors(new ArrayList());
+ response.setWarnings(new ArrayList());
+ Integer rpParallelism = mResourcePlan.getQueryParallelism();
+ if (rpParallelism != null && rpParallelism < 1) {
+ response.addToErrors("Query parallelism should for resource plan be positive. Got: " +
+ rpParallelism);
+ }
+ int totalQueryParallelism = 0;
+ Map<String, PoolData> poolInfo = new HashMap<>();
+ for (MWMPool pool : mResourcePlan.getPools()) {
+ PoolData currentPoolData = getPoolData(poolInfo, pool.getPath());
+ currentPoolData.found = true;
+ String parent = getParentPath(pool.getPath(), "");
+ PoolData parentPoolData = getPoolData(poolInfo, parent);
+ parentPoolData.hasChildren = true;
+ parentPoolData.totalChildrenAllocFraction += pool.getAllocFraction();
+ if (pool.getQueryParallelism() != null && pool.getQueryParallelism() < 1) {
+ response.addToErrors("Invalid query parallelism for pool: " + pool.getPath());
+ } else {
+ totalQueryParallelism += pool.getQueryParallelism();
+ }
+ if (!MetaStoreUtils.isValidSchedulingPolicy(pool.getSchedulingPolicy())) {
+ response.addToErrors("Invalid scheduling policy " + pool.getSchedulingPolicy() +
+ " for pool: " + pool.getPath());
+ }
+ }
+ if (rpParallelism != null) {
+ if (rpParallelism < totalQueryParallelism) {
+ response.addToErrors("Sum of all pools' query parallelism: " + totalQueryParallelism +
+ " exceeds resource plan query parallelism: " + rpParallelism);
+ } else if (rpParallelism != totalQueryParallelism) {
+ response.addToWarnings("Sum of all pools' query parallelism: " + totalQueryParallelism +
+ " is less than resource plan query parallelism: " + rpParallelism);
+ }
+ }
+ for (Map.Entry<String, PoolData> entry : poolInfo.entrySet()) {
+ final PoolData poolData = entry.getValue();
+ final boolean isRoot = entry.getKey().isEmpty();
+ // Special case for root parent
+ if (isRoot) {
+ poolData.found = true;
+ if (!poolData.hasChildren) {
+ response.addToErrors("Root has no children");
+ // TODO: change fractions to use decimal? somewhat brittle
+ } else if (Math.abs(1.0 - poolData.totalChildrenAllocFraction) > 0.00001) {
+ response.addToErrors("Sum of root children pools' alloc fraction should be 1.0 got: " +
+ poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey());
+ }
+ }
+ if (!poolData.found) {
+ response.addToErrors("Pool does not exists but has children: " + entry.getKey());
+ }
+ if (poolData.hasChildren) {
+
+ if (!isRoot && (poolData.totalChildrenAllocFraction - 1.0) > 0.00001) {
+ response.addToErrors("Sum of children pools' alloc fraction should be less than 1 got: "
+ + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey());
+ }
+ }
+ }
+ // trigger and action expressions are not validated here, since counters are not
+ // available and grammar check is there in the language itself.
+ return response;
+ }
+
+ @Override
+ public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns)
+ throws NoSuchObjectException, InvalidObjectException, MetaException {
+ name = normalizeIdentifier(name);
+ Query query = createGetResourcePlanQuery();
+ MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, ns);
+ if (mResourcePlan == null) {
+ throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns);
+ }
+ WMValidateResourcePlanResponse result = getResourcePlanErrors(mResourcePlan);
+ return result;
+ }
+
+ @Override
+ public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException {
+ name = normalizeIdentifier(name);
+ Query query = createGetResourcePlanQuery();
+ MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns));
+ if (resourcePlan == null) {
+ throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns);
+ }
+ pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't?
+ if (resourcePlan.getStatus() == MWMResourcePlan.Status.ACTIVE) {
+ throw new MetaException("Cannot drop an active resource plan");
+ }
+ // First, drop all the dependencies.
+ resourcePlan.setDefaultPool(null);
+ pm.deletePersistentAll(resourcePlan.getTriggers());
+ pm.deletePersistentAll(resourcePlan.getMappings());
+ pm.deletePersistentAll(resourcePlan.getPools());
+ pm.deletePersistent(resourcePlan);
+ }
+
+ @Override
+ public void createWMTrigger(WMTrigger trigger)
+ throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException,
+ MetaException {
+ try {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ trigger.getResourcePlanName(), trigger.getNs(), true);
+ MWMTrigger mTrigger = new MWMTrigger(resourcePlan,
+ normalizeIdentifier(trigger.getTriggerName()), trigger.getTriggerExpression(),
+ trigger.getActionExpression(), null,
+ trigger.isSetIsInUnmanaged() && trigger.isIsInUnmanaged());
+ pm.makePersistent(mTrigger);
+ } catch (Exception e) {
+ checkForConstraintException(e, "Trigger already exists, use alter: ");
+ throw e;
+ }
+ }
+
+ @Override
+ public void alterWMTrigger(WMTrigger trigger)
+ throws NoSuchObjectException, InvalidOperationException, MetaException {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ trigger.getResourcePlanName(), trigger.getNs(), true);
+ MWMTrigger mTrigger = getTrigger(resourcePlan, trigger.getTriggerName());
+ // Update the object.
+ if (trigger.isSetTriggerExpression()) {
+ mTrigger.setTriggerExpression(trigger.getTriggerExpression());
+ }
+ if (trigger.isSetActionExpression()) {
+ mTrigger.setActionExpression(trigger.getActionExpression());
+ }
+ if (trigger.isSetIsInUnmanaged()) {
+ mTrigger.setIsInUnmanaged(trigger.isIsInUnmanaged());
+ }
+ }
+
+ private MWMTrigger getTrigger(MWMResourcePlan resourcePlan, String triggerName)
+ throws NoSuchObjectException {
+ triggerName = normalizeIdentifier(triggerName);
+ Query // Get the MWMTrigger object from DN
+ query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName");
+ query.setUnique(true);
+ MWMTrigger mTrigger = (MWMTrigger) query.execute(resourcePlan, triggerName);
+ if (mTrigger == null) {
+ throw new NoSuchObjectException("Cannot find trigger with name: " + triggerName);
+ }
+ pm.retrieve(mTrigger);
+ return mTrigger;
+ }
+
+ @Override
+ public void dropWMTrigger(String resourcePlanName, String triggerName, String ns)
+ throws NoSuchObjectException, InvalidOperationException, MetaException {
+ resourcePlanName = normalizeIdentifier(resourcePlanName);
+ triggerName = normalizeIdentifier(triggerName);
+
+ Query query = null;
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true);
+ query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName");
+ if (query.deletePersistentAll(resourcePlan, triggerName) != 1) {
+ throw new NoSuchObjectException("Cannot delete trigger: " + triggerName);
+ }
+ }
+
+ @Override
+ public List<WMTrigger> getTriggersForResourcePlan(String resourcePlanName, String ns)
+ throws NoSuchObjectException, MetaException {
+ List<WMTrigger> triggers = new ArrayList();
+ Query query = null;
+ MWMResourcePlan resourcePlan;
+ try {
+ resourcePlan = getMWMResourcePlan(resourcePlanName, ns, false);
+ } catch (InvalidOperationException e) {
+ // Should not happen, edit check is false.
+ throw new RuntimeException(e);
+ }
+ query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp");
+ query.declareParameters("MWMResourcePlan rp");
+ List<MWMTrigger> mTriggers = (List<MWMTrigger>) query.execute(resourcePlan);
+ pm.retrieveAll(mTriggers);
+ if (mTriggers != null) {
+ for (MWMTrigger trigger : mTriggers) {
+ triggers.add(fromMWMTrigger(trigger, resourcePlanName));
+ }
+ }
+ return triggers;
+ }
+
+ private WMTrigger fromMWMTrigger(MWMTrigger mTrigger, String resourcePlanName) {
+ WMTrigger trigger = new WMTrigger();
+ trigger.setResourcePlanName(resourcePlanName);
+ trigger.setTriggerName(mTrigger.getName());
+ trigger.setTriggerExpression(mTrigger.getTriggerExpression());
+ trigger.setActionExpression(mTrigger.getActionExpression());
+ trigger.setIsInUnmanaged(mTrigger.getIsInUnmanaged());
+ trigger.setNs(mTrigger.getResourcePlan().getNs());
+ return trigger;
+ }
+
+ @Override
+ public void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException,
+ InvalidOperationException, MetaException {
+ try {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ pool.getResourcePlanName(), pool.getNs(), true);
+
+ if (!poolParentExists(resourcePlan, pool.getPoolPath())) {
+ throw new NoSuchObjectException("Pool path is invalid, the parent does not exist");
+ }
+ String policy = pool.getSchedulingPolicy();
+ if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) {
+ throw new InvalidOperationException("Invalid scheduling policy " + policy);
+ }
+ MWMPool mPool = new MWMPool(resourcePlan, pool.getPoolPath(), pool.getAllocFraction(),
+ pool.getQueryParallelism(), policy);
+ pm.makePersistent(mPool);
+ } catch (Exception e) {
+ checkForConstraintException(e, "Pool already exists: ");
+ throw e;
+ }
+ }
+
+ @Override
+ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException,
+ NoSuchObjectException, InvalidOperationException, MetaException {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ pool.getResourcePlanName(), pool.getNs(), true);
+ MWMPool mPool = getPool(resourcePlan, poolPath);
+ pm.retrieve(mPool);
+ if (pool.isSetAllocFraction()) {
+ mPool.setAllocFraction(pool.getAllocFraction());
+ }
+ if (pool.isSetQueryParallelism()) {
+ mPool.setQueryParallelism(pool.getQueryParallelism());
+ }
+ if (pool.isSetIsSetSchedulingPolicy() && pool.isIsSetSchedulingPolicy()) {
+ if (pool.isSetSchedulingPolicy()) {
+ String policy = pool.getSchedulingPolicy();
+ if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) {
+ throw new InvalidOperationException("Invalid scheduling policy " + policy);
+ }
+ mPool.setSchedulingPolicy(pool.getSchedulingPolicy());
+ } else {
+ mPool.setSchedulingPolicy(null);
+ }
+ }
+ if (pool.isSetPoolPath() && !pool.getPoolPath().equals(mPool.getPath())) {
+ moveDescendents(resourcePlan, mPool.getPath(), pool.getPoolPath());
+ mPool.setPath(pool.getPoolPath());
+ }
+ }
+
+ private MWMPool getPool(MWMResourcePlan resourcePlan, String poolPath)
+ throws NoSuchObjectException {
+ poolPath = normalizeIdentifier(poolPath);
+ Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path == poolPath");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath");
+ query.setUnique(true);
+ MWMPool mPool = (MWMPool) query.execute(resourcePlan, poolPath);
+ if (mPool == null) {
+ throw new NoSuchObjectException("Cannot find pool: " + poolPath);
+ }
+ pm.retrieve(mPool);
+ return mPool;
+ }
+
+ private void moveDescendents(MWMResourcePlan resourcePlan, String path, String newPoolPath)
+ throws NoSuchObjectException {
+ if (!poolParentExists(resourcePlan, newPoolPath)) {
+ throw new NoSuchObjectException("Pool path is invalid, the parent does not exist");
+ }
+ Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath");
+ List<MWMPool> descPools = (List<MWMPool>) query.execute(resourcePlan, path + ".");
+ pm.retrieveAll(descPools);
+ for (MWMPool pool : descPools) {
+ pool.setPath(newPoolPath + pool.getPath().substring(path.length()));
+ }
+ }
+
+ private String getParentPath(String poolPath, String defValue) {
+ int idx = poolPath.lastIndexOf('.');
+ if (idx == -1) {
+ return defValue;
+ }
+ return poolPath.substring(0, idx);
+ }
+
+ private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) {
+ String parent = getParentPath(poolPath, null);
+ if (parent == null) {
+ return true;
+ }
+ try {
+ getPool(resourcePlan, parent);
+ return true;
+ } catch (NoSuchObjectException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void dropWMPool(String resourcePlanName, String poolPath, String ns)
+ throws NoSuchObjectException, InvalidOperationException, MetaException {
+ poolPath = normalizeIdentifier(poolPath);
+ Query query = null;
+ try {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true);
+ if (resourcePlan.getDefaultPool() != null &&
+ resourcePlan.getDefaultPool().getPath().equals(poolPath)) {
+ throw new InvalidOperationException("Cannot drop default pool of a resource plan");
+ }
+ if (poolHasChildren(resourcePlan, poolPath)) {
+ throw new InvalidOperationException("Cannot drop a pool that has child pools");
+ }
+ query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath");
+ if (query.deletePersistentAll(resourcePlan, poolPath) != 1) {
+ throw new NoSuchObjectException("Cannot delete pool: " + poolPath);
+ }
+ } catch(Exception e) {
+ if (getConstraintException(e) != null) {
+ throw new InvalidOperationException("Please remove all mappings for this pool.");
+ }
+ throw e;
+ }
+ }
+
+ private boolean poolHasChildren(MWMResourcePlan resourcePlan, String poolPath) {
+ Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath");
+ query.setResult("count(this)");
+ query.setUnique(true);
+ Long count = (Long) query.execute(resourcePlan, poolPath + ".");
+ return count != null && count > 0;
+ }
+
+ @Override
+ public void createOrUpdateWMMapping(WMMapping mapping, boolean update)
+ throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException,
+ MetaException {
+ MWMMapping.EntityType entityType = MWMMapping.EntityType.valueOf(mapping.getEntityType().trim().toUpperCase());
+ String entityName = normalizeIdentifier(mapping.getEntityName());
+ Query query = null;
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ mapping.getResourcePlanName(), mapping.getNs(), true);
+ MWMPool pool = null;
+ if (mapping.isSetPoolPath()) {
+ pool = getPool(resourcePlan, mapping.getPoolPath());
+ }
+ if (!update) {
+ MWMMapping mMapping = new MWMMapping(resourcePlan, entityType, entityName, pool,
+ mapping.getOrdering());
+ pm.makePersistent(mMapping);
+ } else {
+ query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type " +
+ "&& entityName == name");
+ query.declareParameters(
+ "MWMResourcePlan rp, java.lang.String type, java.lang.String name");
+ query.setUnique(true);
+ MWMMapping mMapping = (MWMMapping) query.execute(
+ resourcePlan, entityType.toString(), entityName);
+ if (mMapping == null) {
+ throw new NoSuchObjectException("Cannot find mapping for " + entityType + ":" + entityName);
+ }
+ mMapping.setPool(pool);
+ }
+ }
+
+ @Override
+ public void dropWMMapping(WMMapping mapping)
+ throws NoSuchObjectException, InvalidOperationException, MetaException {
+ String entityType = mapping.getEntityType().trim().toUpperCase();
+ String entityName = normalizeIdentifier(mapping.getEntityName());
+ Query query = null;
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(
+ mapping.getResourcePlanName(), mapping.getNs(), true);
+ query = pm.newQuery(MWMMapping.class,
+ "resourcePlan == rp && entityType == type && entityName == name");
+ query.declareParameters("MWMResourcePlan rp, java.lang.String type, java.lang.String name");
+ if (query.deletePersistentAll(resourcePlan, entityType, entityName) != 1) {
+ throw new NoSuchObjectException("Cannot delete mapping.");
+ }
+ }
+
+ @Override
+ public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName,
+ String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException,
+ InvalidOperationException, MetaException {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true);
+ MWMPool pool = getPool(resourcePlan, poolPath);
+ MWMTrigger trigger = getTrigger(resourcePlan, triggerName);
+ pool.getTriggers().add(trigger);
+ trigger.getPools().add(pool);
+ pm.makePersistent(pool);
+ pm.makePersistent(trigger);
+ }
+
+ @Override
+ public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName,
+ String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException {
+ MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true);
+ MWMPool pool = getPool(resourcePlan, poolPath);
+ MWMTrigger trigger = getTrigger(resourcePlan, triggerName);
+ pool.getTriggers().remove(trigger);
+ trigger.getPools().remove(pool);
+ pm.makePersistent(pool);
+ pm.makePersistent(trigger);
+ }
+
+ @Override
+ public void addRuntimeStat(RuntimeStat stat) throws MetaException {
+ LOG.debug("runtimeStat: {}", stat);
+ MRuntimeStat mStat = MRuntimeStat.fromThrift(stat);
+ pm.makePersistent(mStat);
+ }
+
+ @Override
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
+ if (maxRetainSecs < 0) {
+ LOG.warn("runtime stats retention is disabled");
+ return 0;
+ }
+ int maxCreateTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs;
+ Query q = pm.newQuery(MRuntimeStat.class);
+ q.setFilter("createTime <= maxCreateTime");
+ q.declareParameters("int maxCreateTime");
+ long deleted = q.deletePersistentAll(maxCreateTime);
+ return (int) deleted;
+ }
+
+ @Override
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
+ return getMRuntimeStats(maxEntries, maxCreateTime);
+ }
+
+ private List<RuntimeStat> getMRuntimeStats(int maxEntries, int maxCreateTime) {
+ try (QueryWrapper query = new QueryWrapper(pm.newQuery(MRuntimeStat.class))) {
+ query.setOrdering("createTime descending");
+ if (maxCreateTime > 0) {
+ query.setFilter("createTime < " + maxCreateTime);
+ }
+ if (maxEntries < 0) {
+ maxEntries = Integer.MAX_VALUE;
+ }
+ List<RuntimeStat> ret = new ArrayList<>();
+ List<MRuntimeStat> res = (List<MRuntimeStat>) query.execute();
+ int totalEntries = 0;
+ for (MRuntimeStat mRuntimeStat : res) {
+ pm.retrieve(mRuntimeStat);
+ totalEntries += mRuntimeStat.getWeight();
+ ret.add(MRuntimeStat.toThrift(mRuntimeStat));
+ if (totalEntries >= maxEntries) {
+ break;
+ }
+ }
+ return ret;
+ }
+ }
+
+ private Configuration conf;
Review Comment:
nit, `private Configuration conf` is declared near the end of the class, after most methods. Could we move it near other fields at the top?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java:
##########
@@ -540,22 +434,38 @@ public <T> T unwrap(Class<T> iface) {
cachedImpls.put(iface, simpl);
}
List<Query> openQueries = new LinkedList<>();
- if (simpl instanceof RawStoreAware rsa) {
+ if (simpl instanceof RawStoreBundle rsa) {
Review Comment:
nit, Could we rename `rsa` to `rsb`, `bundle`, or similar?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java:
##########
@@ -0,0 +1,1226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metastore.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import javax.jdo.Query;
+import javax.jdo.datastore.JDOConnection;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.Batchable;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.Deadline;
+import org.apache.hadoop.hive.metastore.PersistenceManagerProvider;
+import org.apache.hadoop.hive.metastore.QueryWrapper;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionFilterMode;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats;
+import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats;
+import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql;
+import org.apache.hadoop.hive.metastore.metastore.GetHelper;
+import org.apache.hadoop.hive.metastore.metastore.GetListHelper;
+import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle;
+import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore;
+import org.apache.hadoop.hive.metastore.metastore.iface.TableStore;
+import org.apache.hadoop.hive.metastore.model.MPartition;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MTable;
+import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.utils.RetryingExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING;
+import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams;
+import static org.apache.hadoop.hive.metastore.ObjectStore.isCurrentStatsValidForTheQuery;
+import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers;
+
+public class ColStatsStoreImpl extends RawStoreBundle implements ColStatsStore {
+ private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class);
+
+ private DatabaseProduct dbType;
+ protected int batchSize = NO_BATCHING;
+ private boolean areTxnStatsSupported = false;
+ private Configuration conf;
+ private String schema;
+ private SQLGenerator sqlGenerator;
+
+ @Override
+ public void setBaseStore(RawStore store) {
+ super.setBaseStore(store);
+ this.dbType = PersistenceManagerProvider.getDatabaseProduct();
+ this.batchSize = MetastoreConf.getIntVar(store.getConf(),
+ MetastoreConf.ConfVars.RAWSTORE_PARTITION_BATCH_SIZE);
+ this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(),
+ MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED);
+ this.conf = store.getConf();
+ this.sqlGenerator = new SQLGenerator(dbType, conf);
+ String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema");
+ this.schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null);
+ }
+
+ @Override
+ public List<TableName> getTableNamesWithStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName> (this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getTableNamesWithStats();
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+ }.run(false);
+ }
+
+ @Override
+ public Map<String, List<String>> getPartitionColsWithStats(TableName tableName)
+ throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new GetHelper<TableName, Map<String, List<String>>>(this, null) {
+ @Override
+ protected Map<String, List<String>> getSqlResult() throws MetaException {
+ try {
+ return getDirectSql().getColAndPartNamesWithStats(catName, dbName, tblName);
+ } catch (Throwable ex) {
+ LOG.error("DirectSQL failed", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, List<String>> getJdoResult() throws MetaException {
+ throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement?
+ }
+
+ @Override
+ protected String describeResult() {
+ return results.size() + " partitions";
+ }
+ }.run(false);
+ }
+
+ @Override
+ public List<TableName> getAllTableNamesForStats() throws MetaException, NoSuchObjectException {
+ return new GetListHelper<TableName, TableName>(this, null) {
+ @Override
+ protected List<TableName> getSqlResult() throws MetaException {
+ return getDirectSql().getAllTableNamesForStats();
+ }
+
+ @Override
+ protected List<TableName> getJdoResult() throws MetaException {
+ List<TableName> result = new ArrayList<>();
+ String paramStr = "", whereStr = "";
+ for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) {
+ if (i != 0) {
+ paramStr += ", ";
+ whereStr += "||";
+ }
+ paramStr += "java.lang.String tt" + i;
+ whereStr += " tableType == tt" + i;
+ }
+ Query query = pm.newQuery(MTable.class, whereStr);
+ query.declareParameters(paramStr);
+ Collection<MTable> tbls =
+ (Collection<MTable>) query.executeWithArray(MetaStoreDirectSql.STATS_TABLE_TYPES);
+ pm.retrieveAll(tbls);
+ for (MTable tbl : tbls) {
+ result.add(new TableName(
+ tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName()));
+ }
+ return result;
+ }
+ }.run(false);
+ }
+
+ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj,
+ MTableColumnStatistics oldStats) throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ String colName = mStatsObj.getColName();
+
+ LOG.info("Updating table level column statistics for table={} colName={}",
+ Warehouse.getCatalogQualifiedTableName(table), colName);
+ validateTableCols(table, Lists.newArrayList(colName));
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ private void writeMPartitionColumnStatistics(Table table, Partition partition,
+ MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) {
+ String catName = mStatsObj.getPartition().getTable().getDatabase().getCatalogName();
+ String dbName = mStatsObj.getPartition().getTable().getDatabase().getName();
+ String tableName = mStatsObj.getPartition().getTable().getTableName();
+ String partName = mStatsObj.getPartition().getPartitionName();
+ String colName = mStatsObj.getColName();
+
+ Preconditions.checkState(this.baseStore.isActiveTransaction());
+
+ LOG.info("Updating partition level column statistics for table=" +
+ TableName.getQualified(catName, dbName, tableName) +
+ " partName=" + partName + " colName=" + colName);
+
+ boolean foundCol = false;
+ List<FieldSchema> colList = partition.getSd().getCols();
+ for (FieldSchema col : colList) {
+ if (col.getName().equals(mStatsObj.getColName())) {
+ foundCol = true;
+ break;
+ }
+ }
+
+ if (!foundCol) {
+ LOG.warn("Column " + colName + " for which stats gathering is requested doesn't exist.");
+ }
+
+ if (oldStats != null) {
+ StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
+ } else {
+ pm.makePersistent(mStatsObj);
+ }
+ }
+
+ @Override
+ public Map<String, String> updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ long start = System.currentTimeMillis();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ // DataNucleus objects get detached all over the place for no (real) reason.
+ // So let's not use them anywhere unless absolutely necessary.
+ MTable mTable = baseStore.ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName());
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> transactionHelper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ transactionHelper.setTransactionSavePoint(savePoint);
+ executePlainSQL(
+ sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()),
+ true,
+ exception -> {
+ transactionHelper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mTable);
+ Table table = baseStore.unwrap(TableStore.class).getTable(new TableName(catName, statsDesc.getDbName(),
+ statsDesc.getTableName()), null, -1);
+ List<String> colNames = new ArrayList<>();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+
+ Map<String, MTableColumnStatistics> oldStats = Maps.newHashMap();
+ List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, colStats.getEngine());
+ for (MTableColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc,
+ statsObj, colStats.getEngine());
+ writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName()));
+ // There is no need to add colname again, otherwise we will get duplicate colNames.
+ }
+
+ // Set the table properties
+ // No need to check again if it exists.
+ String dbname = table.getDbName();
+ String name = table.getTableName();
+ MTable oldt = mTable;
+ Map<String, String> newParams = new HashMap<>(table.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters());
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams,
+ writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(oldt.getParameters(), oldt.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name);
+ }
+ oldt.setWriteId(writeId);
+ }
+ }
+ oldt.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+
+ LOG.debug("{} updateTableColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ long start = System.currentTimeMillis();
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
+ MPartition mPartition =
+ baseStore.ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals);
+ if (mPartition == null) {
+ throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
+ }
+
+ List<String> colNames = new ArrayList<>();
+ for(ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+ int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS);
+ GetHelper<TableName, String> helper =
+ createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+ String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime();
+ helper.setTransactionSavePoint(savePoint);
+ executePlainSQL(sqlGenerator.addForUpdateNoWait(
+ "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()),
+ true,
+ exception -> {
+ helper.rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.set(exception);
+ });
+ if (exceptionRef.get() != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.get());
+ }
+ pm.refresh(mPartition);
+ Partition partition = baseStore.unwrap(TableStore.class)
+ .getPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getValues(), null);
+ Map<String, MPartitionColumnStatistics> oldStats = Maps.newHashMap();
+ List<MPartitionColumnStatistics> stats =
+ getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine());
+ for (MPartitionColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition,
+ statsDesc, statsObj, colStats.getEngine());
+ writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName()));
+ }
+
+ Map<String, String> newParams = new HashMap<>(mPartition.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(table);
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ } else {
+ String errorMsg = verifyStatsChangeCtx(
+ TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(),
+ newParams, writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the current value.
+ StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName());
+ }
+ mPartition.setWriteId(writeId);
+ }
+ }
+ mPartition.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+ .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
+ LOG.debug("{} updatePartitionColumnStatistics took {}ms",
+ new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()),
+ System.currentTimeMillis() - start);
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
+ Map<String, ColumnStatistics> partColStatsMap,
+ Table tbl,
+ List<TransactionalMetaStoreEventListener> listeners,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+
+ return new GetHelper<TableName, Map<String, Map<String, String>>>(this, null) {
+ @Override
+ protected String describeResult() {
+ return "Map of partition key to column stats if successful";
+ }
+ @Override
+ protected Map<String, Map<String, String>> getSqlResult()
+ throws MetaException {
+ return getDirectSql().updatePartitionColumnStatisticsBatch(partColStatsMap, tbl,
+ listeners, validWriteIds, writeId);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected Map<String, Map<String, String>> getJdoResult()
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled");
+ }
+ }.run(false);
+ }
+
+ private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, String engine)
+ throws MetaException {
+
+ Preconditions.checkState(baseStore.isActiveTransaction());
+
+ if (colNames.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ validateTableCols(table, colNames);
+
+ List<MTableColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ result =
+ Batchable.runBatched(batchSize, colNames, new Batchable<String, MTableColumnStatistics>() {
+ @Override
+ public List<MTableColumnStatistics> run(List<String> input)
+ throws MetaException {
+ StringBuilder filter =
+ new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && (");
+ StringBuilder paramStr = new StringBuilder(
+ "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4");
+ Object[] params = new Object[input.size() + 4];
+ params[0] = table.getTableName();
+ params[1] = table.getDbName();
+ params[2] = table.getCatName();
+ params[3] = engine;
+ for (int i = 0; i < input.size(); ++i) {
+ filter.append((i == 0) ? "" : " || ").append("colName == c").append(i);
+ paramStr.append(", java.lang.String c").append(i);
+ params[i + 4] = input.get(i);
+ }
+ filter.append(")");
+ query.setFilter(filter.toString());
+ query.declareParameters(paramStr.toString());
+ List<MTableColumnStatistics> paritial = (List<MTableColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(paritial);
+ return paritial;
+ }
+ });
+
+ if (result.size() > colNames.size()) {
+ throw new MetaException("Unexpected " + result.size() + " statistics for "
+ + colNames.size() + " columns");
+ }
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public List<ColumnStatistics> getTableColumnStatistics(
+ TableName tblName,
+ List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<ColumnStatistics> result = new ArrayList<>();
+ Query query = pm.newQuery(MTableColumnStatistics.class);
+ query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ ColumnStatistics cs = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ return getTableColumnStatisticsInternal(tableName, colNames, engine);
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ TableName tableName,
+ List<String> colNames,
+ String engine,
+ String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, set No to the compliance flag.
+ Boolean isCompliant = null;
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ if (writeIdList != null) {
+ MTable table = baseStore.ensureGetMTable(catName, dbName, tblName);
+ isCompliant = !TxnUtils.isTransactionalTable(table.getParameters())
+ || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table.getParameters(), table.getWriteId(), writeIdList, false));
+ }
+ ColumnStatistics stats = getTableColumnStatisticsInternal(
+ new TableName(catName, dbName, tblName), colNames, engine);
+ if (stats != null && isCompliant != null) {
+ stats.setIsStatsCompliant(isCompliant);
+ }
+ return stats;
+ }
+
+ protected ColumnStatistics getTableColumnStatisticsInternal(
+ TableName tableName, final List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetStatHelper(tableName, this) {
+ @Override
+ protected ColumnStatistics getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected ColumnStatistics getJdoResult() throws MetaException {
+
+ List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, engine);
+ if (mStats.isEmpty()) {
+ return null;
+ }
+ // LastAnalyzed is stored per column, but thrift object has it per
+ // multiple columns. Luckily, nobody actually uses it, so we will set to
+ // lowest value of all columns for now.
+ ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
+ List<ColumnStatisticsObj> statObjs = new ArrayList<>(mStats.size());
+ for (MTableColumnStatistics mStat : mStats) {
+ if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
+ desc.setLastAnalyzed(mStat.getLastAnalyzed());
+ }
+ statObjs.add(StatObjectConverter.getColumnStatisticsObj(mStat, enableBitVector, enableKll));
+ Deadline.checkTimeout();
+ }
+ ColumnStatistics colStat = new ColumnStatistics(desc, statObjs);
+ colStat.setEngine(engine);
+ return colStat;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<List<ColumnStatistics>> getPartitionColumnStatistics(TableName tblName,
+ List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
+ String catName = normalizeIdentifier(tblName.getCat());
+ String dbName = normalizeIdentifier(tblName.getDb());
+ String tableName = normalizeIdentifier(tblName.getTable());
+ // Note: this will get stats without verifying ACID.
+ List<List<ColumnStatistics>> result = new ArrayList<>();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setResult("DISTINCT engine");
+ Collection names = (Collection) query.execute(tableName, dbName, catName);
+ List<String> engines = new ArrayList<>();
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ engines.add((String) i.next());
+ }
+ for (String e : engines) {
+ List<ColumnStatistics> cs = getPartitionColumnStatisticsInternal(
+ new TableName(catName, dbName, tableName), partNames, colNames, e);
+ if (cs != null) {
+ result.add(cs);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(TableName tableName,
+ List<String> partNames, List<String> colNames, String engine) throws MetaException, NoSuchObjectException {
+ // Note: this will get stats without verifying ACID.
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ return getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ TableName tableName,
+ List<String> partNames, List<String> colNames,
+ String engine, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) {
+ LOG.debug("PartNames and/or ColNames are empty");
+ return Collections.emptyList();
+ }
+ List<ColumnStatistics> allStats = getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine);
+ if (writeIdList != null) {
+ if (!areTxnStatsSupported) {
+ for (ColumnStatistics cs : allStats) {
+ cs.setIsStatsCompliant(false);
+ }
+ } else {
+ // TODO: this could be improved to get partitions in bulk
+ for (ColumnStatistics cs : allStats) {
+ MPartition mpart = baseStore.ensureGetMPartition(tableName,
+ Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName()));
+ if (mpart == null
+ || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) {
+ if (mpart != null) {
+ LOG.debug("The current metastore transactional partition column statistics for {} {} "
+ + "(write ID {}) are not valid for current query ({})", tableName,
+ mpart.getPartitionName(), mpart.getWriteId(), writeIdList);
+ }
+ cs.setIsStatsCompliant(false);
+ } else {
+ cs.setIsStatsCompliant(true);
+ }
+ }
+ }
+ }
+ return allStats;
+ }
+
+ protected List<ColumnStatistics> getPartitionColumnStatisticsInternal(
+ TableName tableName, final List<String> partNames, final List<String> colNames,
+ String engine) throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetListHelper<TableName, ColumnStatistics>(this, tableName) {
+ @Override
+ protected List<ColumnStatistics> getSqlResult() throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return new DirectSqlAggrStats(pm, conf, schema).getPartitionStats(
+ catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll);
+ }
+ @Override
+ protected List<ColumnStatistics> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ List<MPartitionColumnStatistics> mStats =
+ getMPartitionColumnStatistics(getTable(), partNames, colNames, engine);
+ List<ColumnStatistics> result = new ArrayList<>(Math.min(mStats.size(), partNames.size()));
+ String lastPartName = null;
+ List<ColumnStatisticsObj> curList = null;
+ ColumnStatisticsDesc csd = null;
+ for (int i = 0; i <= mStats.size(); ++i) {
+ boolean isLast = i == mStats.size();
+ MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
+ String partName = isLast ? null : mStatsObj.getPartition().getPartitionName();
+ if (isLast || !partName.equals(lastPartName)) {
+ if (i != 0) {
+ ColumnStatistics colStat = new ColumnStatistics(csd, curList);
+ colStat.setEngine(engine);
+ result.add(colStat);
+ }
+ if (isLast) {
+ continue;
+ }
+ csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
+ curList = new ArrayList<>(colNames.size());
+ }
+ curList.add(StatObjectConverter.getColumnStatisticsObj(mStatsObj, enableBitVector, enableKll));
+ lastPartName = partName;
+ Deadline.checkTimeout();
+ }
+ return result;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames,
+ String engine, String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, return null.
+ if (writeIdList != null) {
+ if (partNames == null || partNames.isEmpty()) {
+ return null;
+ }
+
+ Table table = baseStore.unwrap(TableStore.class).getTable(tableName, null, -1);
+ boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
+ if (isTxn && !areTxnStatsSupported) {
+ return null;
+ }
+ GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+ fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+ fs.setFilters(partNames);
+ GetProjectionsSpec ps = new GetProjectionsSpec();
+ ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%');
+ ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+ List<Partition> parts = baseStore.unwrap(TableStore.class)
+ .getPartitionSpecsByFilterAndProjection(table, ps, fs);
+
+ // Loop through the given "partNames" list
+ // checking isolation-level-compliance of each partition column stats.
+ for (Partition part : parts) {
+
+ if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ LOG.debug("The current metastore transactional partition column "
+ + "statistics for {} {} is not valid for the current query",
+ tableName, partName);
+ return null;
+ }
+ }
+ }
+ return get_aggr_stats_for(tableName, partNames, colNames, engine);
+ }
+
+ @Override
+ public AggrStats get_aggr_stats_for(TableName tableName,
+ final List<String> partNames, final List<String> colNames, String engine)
+ throws MetaException, NoSuchObjectException {
+ final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(conf,
+ MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION);
+ final double ndvTuner = MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER);
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, AggrStats>(this, tableName) {
+ @Override
+ protected AggrStats getSqlResult()
+ throws MetaException {
+ String catName = normalizeIdentifier(tableName.getCat());
+ String dbName = normalizeIdentifier(tableName.getDb());
+ String tblName = normalizeIdentifier(tableName.getTable());
+ return getDirectSql().aggrColStatsForPartitions(catName, dbName, tblName, partNames,
+ colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected AggrStats getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using
+ // directSql, do it. No point in failing back to slow path here.
+ throw new MetaException("Jdo path is not implemented for stats aggr.");
+ }
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(true);
+ }
+
+ @Override
+ public List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName)
+ throws MetaException, NoSuchObjectException {
+ final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
+ return new GetHelper<TableName, List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(this, null) {
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getSqlResult() throws MetaException {
+ return new DirectSqlAggrStats(pm, conf, schema)
+ .getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll);
+ }
+
+ @Override
+ protected boolean canUseJdoQuery() throws MetaException {
+ return false;
+ }
+
+ @Override
+ protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getJdoResult()
+ throws MetaException, NoSuchObjectException {
+ // This is fast path for query optimizations, if we can find this info
+ // quickly using directSql, do it. No point in failing back to slow path
+ // here.
+ throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase.");
+ }
+
+ @Override
+ protected String describeResult() {
+ return null;
+ }
+ }.run(false);
+ }
+
+ private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(Table table, List<String> partNames,
+ List<String> colNames, String engine) throws MetaException {
+ // We are not going to verify SD for each partition. Just verify for the
+ // table. TODO: we need verify the partition column instead
+ try {
+ validateTableCols(table, colNames);
+ } catch (MetaException me) {
+ LOG.warn("The table does not have the same column definition as its partition.");
+ }
+ List<MPartitionColumnStatistics> result = Collections.emptyList();
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4";
+ String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && " +
+ "partition.table.database.catalogName == t3 && engine == t4 && (";
+ Object[] params = new Object[colNames.size() + partNames.size() + 4];
+ int i = 0;
+ params[i++] = table.getTableName();
+ params[i++] = table.getDbName();
+ params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf);
+ params[i++] = engine;
+ int firstI = i;
+ for (String s : partNames) {
+ filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i;
+ paramStr += ", java.lang.String p" + i;
+ params[i++] = s;
+ }
+ filter += ") && (";
+ firstI = i;
+ for (String s : colNames) {
+ filter += ((i == firstI) ? "" : " || ") + "colName == c" + i;
+ paramStr += ", java.lang.String c" + i;
+ params[i++] = s;
+ }
+ filter += ")";
+ query.setFilter(filter);
+ query.declareParameters(paramStr);
+ query.setOrdering("partition.partitionName ascending");
+ result = (List<MPartitionColumnStatistics>) query.executeWithArray(params);
+ pm.retrieveAll(result);
+ return new ArrayList<>(result);
+ }
+
+ @Override
+ public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ String catName = tn.getCat();
+ String dbName = tn.getDb();
+ String tableName = tn.getTable();
+
+ dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME);
+ catName = normalizeIdentifier(catName);
+ if (tableName == null) {
+ throw new RuntimeException("Table name is null.");
+ }
+ MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName);
+ Query query = pm.newQuery(MPartitionColumnStatistics.class);
+ String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4";
+ String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4";
+
+ query.setFilter(filter);
+ query.declareParameters(parameters);
+ query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName),
+ normalizeIdentifier(catName));
+
+ new GetHelper<TableName, Integer>(this, tn) {
+ private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown();
+
+ @Override
+ protected String describeResult() {
+ return "Partition count";
+ }
+
+ @Override
+ protected Integer getSqlResult() throws MetaException {
+ getDirectSql().deleteColumnStatsState(getTable().getId());
+ return 0;
+ }
+
+ @Override
+ protected Integer getJdoResult() throws MetaException, NoSuchObjectException {
+ try {
+ List<Partition> parts = baseStore.unwrap(TableStore.class).getPartitions(tn,
Review Comment:
nit, Could we changes the line break something like this?
```suggestion
List<Partition> parts = baseStore.unwrap(TableStore.class)
.getPartitions(tn, GetPartitionsArgs.getAllPartitions());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org