You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/18 22:03:43 UTC
[52/67] [abbrv] hive git commit: HIVE-19532 : 04 patch (Steve Yeom)
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index da41e6e..626e103 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -791,6 +791,50 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
+ public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
+ String dbName, String tableName, List<String> partNames, List<String> colNames,
+ long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ return getPartitionColumnStatistics(getDefaultCatalog(conf), dbName, tableName,
+ partNames, colNames, txnId, validWriteIdList);
+ }
+
+ @Override
+ public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
+ String catName, String dbName, String tableName, List<String> partNames,
+ List<String> colNames, long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ PartitionsStatsRequest rqst = new PartitionsStatsRequest(dbName, tableName, colNames,
+ partNames);
+ rqst.setCatName(catName);
+ rqst.setTxnId(txnId);
+ rqst.setValidWriteIdList(validWriteIdList);
+ return client.get_partitions_statistics_req(rqst).getPartStats();
+ }
+
+ @Override
+ public AggrStats getAggrColStatsFor(String dbName, String tblName, List<String> colNames,
+ List<String> partNames, long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ return getAggrColStatsFor(getDefaultCatalog(conf), dbName, tblName, colNames,
+ partNames, txnId, writeIdList); }
+
+ @Override
+ public AggrStats getAggrColStatsFor(String catName, String dbName, String tblName, List<String> colNames,
+ List<String> partNames, long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ if (colNames.isEmpty() || partNames.isEmpty()) {
+ LOG.debug("Columns is empty or partNames is empty : Short-circuiting stats eval on client side.");
+ return new AggrStats(new ArrayList<>(),0); // Nothing to aggregate
+ }
+ PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partNames);
+ req.setCatName(catName);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(writeIdList);
+ return client.get_aggr_stats_for(req);
+ }
+
+ @Override
public List<Partition> exchange_partitions(Map<String, String> partitionSpecs, String sourceCat,
String sourceDb, String sourceTable, String destCat,
String destDb, String destTableName) throws TException {
@@ -1584,6 +1628,14 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
+ public Table getTable(String dbname, String name,
+ long txnId, String validWriteIdList)
+ throws MetaException, TException, NoSuchObjectException{
+ return getTable(getDefaultCatalog(conf), dbname, name,
+ txnId, validWriteIdList);
+ };
+
+ @Override
public Table getTable(String catName, String dbName, String tableName) throws TException {
GetTableRequest req = new GetTableRequest(dbName, tableName);
req.setCatName(catName);
@@ -1593,6 +1645,18 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
+ public Table getTable(String catName, String dbName, String tableName,
+ long txnId, String validWriteIdList) throws TException {
+ GetTableRequest req = new GetTableRequest(dbName, tableName);
+ req.setCatName(catName);
+ req.setCapabilities(version);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(validWriteIdList);
+ Table t = client.get_table_req(req).getTable();
+ return deepCopy(filterHook.filterTable(t));
+ }
+
+ @Override
public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
throws TException {
return getTableObjectsByName(getDefaultCatalog(conf), dbName, tableNames);
@@ -1821,21 +1885,42 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void alter_partitions(String dbName, String tblName, List<Partition> newParts)
throws TException {
- alter_partitions(getDefaultCatalog(conf), dbName, tblName, newParts, null);
+ alter_partitions(
+ getDefaultCatalog(conf), dbName, tblName, newParts, null, -1, null);
}
@Override
public void alter_partitions(String dbName, String tblName, List<Partition> newParts,
EnvironmentContext environmentContext) throws TException {
- alter_partitions(getDefaultCatalog(conf), dbName, tblName, newParts, environmentContext);
+ alter_partitions(
+ getDefaultCatalog(conf), dbName, tblName, newParts, environmentContext, -1, null);
+ }
+
+ @Override
+ public void alter_partitions(String dbName, String tblName, List<Partition> newParts,
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException, TException {
+ //client.alter_partition_with_environment_context(getDefaultCatalog(conf),
+ // dbName, tblName, newParts, environmentContext);
+ alter_partitions(getDefaultCatalog(conf),
+ dbName, tblName, newParts, environmentContext, txnId, writeIdList);
+
}
@Override
public void alter_partitions(String catName, String dbName, String tblName,
List<Partition> newParts,
- EnvironmentContext environmentContext) throws TException {
- client.alter_partitions_with_environment_context(prependCatalogToDbName(catName, dbName, conf),
- tblName, newParts, environmentContext);
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList) throws TException {
+ AlterPartitionsRequest req = new AlterPartitionsRequest();
+ req.setDbName(prependCatalogToDbName(catName, dbName, conf));
+ req.setTableName(tblName);
+ req.setPartitions(newParts);
+ req.setEnvironmentContext(environmentContext);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(writeIdList);
+ client.alter_partitions_with_environment_context(req);
}
@Override
@@ -1967,6 +2052,28 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
+ public List<ColumnStatisticsObj> getTableColumnStatistics(String dbName, String tableName,
+ List<String> colNames,
+ long txnId,
+ String validWriteIdList) throws TException {
+ return getTableColumnStatistics(getDefaultCatalog(conf), dbName, tableName, colNames,
+ txnId, validWriteIdList);
+ }
+
+ @Override
+ public List<ColumnStatisticsObj> getTableColumnStatistics(String catName, String dbName,
+ String tableName,
+ List<String> colNames,
+ long txnId,
+ String validWriteIdList) throws TException {
+ TableStatsRequest rqst = new TableStatsRequest(dbName, tableName, colNames);
+ rqst.setCatName(catName);
+ rqst.setTxnId(txnId);
+ rqst.setValidWriteIdList(validWriteIdList);
+ return client.get_table_statistics_req(rqst).getTableStats();
+ }
+
+ @Override
public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
String dbName, String tableName, List<String> partNames, List<String> colNames)
throws TException {
@@ -3319,4 +3426,5 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
req.setMaxCreateTime(maxCreateTime);
return client.get_runtime_stats(req);
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
index 29c98d1..3a65f77 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
@@ -90,6 +90,11 @@ public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable {
Table get_table_core(final String catName, final String dbname, final String name)
throws MetaException, NoSuchObjectException;
+ Table get_table_core(final String catName, final String dbname,
+ final String name, final long txnId,
+ final String writeIdList)
+ throws MetaException, NoSuchObjectException;
+
/**
* Get a list of all transactional listeners.
* @return list of listeners.
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index bc09076..c4cd8b4 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -712,6 +712,10 @@ public interface IMetaStoreClient {
Table getTable(String dbName, String tableName) throws MetaException,
TException, NoSuchObjectException;
+ Table getTable(String dbName, String tableName,
+ long txnId, String validWriteIdList)
+ throws MetaException, TException, NoSuchObjectException;
+
/**
* Get a table object.
* @param catName catalog the table is in.
@@ -723,6 +727,8 @@ public interface IMetaStoreClient {
*/
Table getTable(String catName, String dbName, String tableName) throws MetaException, TException;
+ Table getTable(String catName, String dbName, String tableName,
+ long txnId, String validWriteIdList) throws TException;
/**
* Get tables as objects (rather than just fetching their names). This is more expensive and
* should only be used if you actually need all the information about the tables.
@@ -2125,6 +2131,11 @@ public interface IMetaStoreClient {
EnvironmentContext environmentContext)
throws InvalidOperationException, MetaException, TException;
+ void alter_partitions(String dbName, String tblName, List<Partition> newParts,
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException, TException;
+
/**
* updates a list of partitions
* @param catName catalog name.
@@ -2144,7 +2155,7 @@ public interface IMetaStoreClient {
default void alter_partitions(String catName, String dbName, String tblName,
List<Partition> newParts)
throws InvalidOperationException, MetaException, TException {
- alter_partitions(catName, dbName, tblName, newParts, null);
+ alter_partitions(catName, dbName, tblName, newParts, null,-1, null);
}
/**
@@ -2165,7 +2176,8 @@ public interface IMetaStoreClient {
* if error in communicating with metastore server
*/
void alter_partitions(String catName, String dbName, String tblName, List<Partition> newParts,
- EnvironmentContext environmentContext)
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList)
throws InvalidOperationException, MetaException, TException;
/**
@@ -2346,6 +2358,12 @@ public interface IMetaStoreClient {
List<ColumnStatisticsObj> getTableColumnStatistics(String dbName, String tableName,
List<String> colNames) throws NoSuchObjectException, MetaException, TException;
+ List<ColumnStatisticsObj> getTableColumnStatistics(String dbName, String tableName,
+ List<String> colNames,
+ long txnId,
+ String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException;
+
/**
* Get the column statistics for a set of columns in a table. This should only be used for
* non-partitioned tables. For partitioned tables use
@@ -2363,6 +2381,11 @@ public interface IMetaStoreClient {
List<String> colNames)
throws NoSuchObjectException, MetaException, TException;
+ List<ColumnStatisticsObj> getTableColumnStatistics(String catName, String dbName, String tableName,
+ List<String> colNames,
+ long txnId,
+ String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException;
/**
* Get the column statistics for a set of columns in a partition.
* @param dbName database name
@@ -2379,6 +2402,11 @@ public interface IMetaStoreClient {
String tableName, List<String> partNames, List<String> colNames)
throws NoSuchObjectException, MetaException, TException;
+ Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(String dbName,
+ String tableName, List<String> partNames, List<String> colNames,
+ long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException;
+
/**
* Get the column statistics for a set of columns in a partition.
* @param catName catalog name
@@ -2396,6 +2424,11 @@ public interface IMetaStoreClient {
String catName, String dbName, String tableName, List<String> partNames, List<String> colNames)
throws NoSuchObjectException, MetaException, TException;
+ Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
+ String catName, String dbName, String tableName,
+ List<String> partNames, List<String> colNames,
+ long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException;
/**
* Delete partition level column statistics given dbName, tableName, partName and colName, or
* all columns in a partition.
@@ -3237,6 +3270,10 @@ public interface IMetaStoreClient {
AggrStats getAggrColStatsFor(String dbName, String tblName,
List<String> colNames, List<String> partName) throws NoSuchObjectException, MetaException, TException;
+ AggrStats getAggrColStatsFor(String dbName, String tblName,
+ List<String> colNames, List<String> partName,
+ long txnId, String writeIdList) throws NoSuchObjectException, MetaException, TException;
+
/**
* Get aggregated column stats for a set of partitions.
* @param catName catalog name
@@ -3253,6 +3290,10 @@ public interface IMetaStoreClient {
List<String> colNames, List<String> partNames)
throws NoSuchObjectException, MetaException, TException;
+ AggrStats getAggrColStatsFor(String catName, String dbName, String tblName,
+ List<String> colNames, List<String> partNames,
+ long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException, TException;
/**
* Set table or partition column statistics.
* @param request request object, contains all the table, partition, and statistics information
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index e99f888..28426b2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -80,140 +80,29 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.DatabaseName;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.common.*;
import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesRow;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.RuntimeStat;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SchemaCompatibility;
-import org.apache.hadoop.hive.metastore.api.SchemaType;
-import org.apache.hadoop.hive.metastore.api.SchemaValidation;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionState;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SerdeType;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMMapping;
-import org.apache.hadoop.hive.metastore.api.WMNullablePool;
-import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMPool;
-import org.apache.hadoop.hive.metastore.api.WMPoolTrigger;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus;
-import org.apache.hadoop.hive.metastore.api.WMTrigger;
-import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
-import org.apache.hadoop.hive.metastore.model.MCatalog;
-import org.apache.hadoop.hive.metastore.model.MColumnDescriptor;
-import org.apache.hadoop.hive.metastore.model.MConstraint;
-import org.apache.hadoop.hive.metastore.model.MCreationMetadata;
-import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
-import org.apache.hadoop.hive.metastore.model.MDatabase;
-import org.apache.hadoop.hive.metastore.model.MDelegationToken;
-import org.apache.hadoop.hive.metastore.model.MFieldSchema;
-import org.apache.hadoop.hive.metastore.model.MFunction;
-import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege;
-import org.apache.hadoop.hive.metastore.model.MISchema;
-import org.apache.hadoop.hive.metastore.model.MMasterKey;
-import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties;
-import org.apache.hadoop.hive.metastore.model.MNotificationLog;
-import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
-import org.apache.hadoop.hive.metastore.model.MOrder;
-import org.apache.hadoop.hive.metastore.model.MPartition;
-import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege;
-import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
-import org.apache.hadoop.hive.metastore.model.MPartitionEvent;
-import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege;
-import org.apache.hadoop.hive.metastore.model.MResourceUri;
-import org.apache.hadoop.hive.metastore.model.MRole;
-import org.apache.hadoop.hive.metastore.model.MRoleMap;
-import org.apache.hadoop.hive.metastore.model.MRuntimeStat;
-import org.apache.hadoop.hive.metastore.model.MSchemaVersion;
-import org.apache.hadoop.hive.metastore.model.MSerDeInfo;
-import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
-import org.apache.hadoop.hive.metastore.model.MStringList;
-import org.apache.hadoop.hive.metastore.model.MTable;
-import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
-import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
-import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
-import org.apache.hadoop.hive.metastore.model.MType;
-import org.apache.hadoop.hive.metastore.model.MVersionTable;
-import org.apache.hadoop.hive.metastore.model.MWMMapping;
+import org.apache.hadoop.hive.metastore.model.*;
import org.apache.hadoop.hive.metastore.model.MWMMapping.EntityType;
-import org.apache.hadoop.hive.metastore.model.MWMPool;
-import org.apache.hadoop.hive.metastore.model.MWMResourcePlan;
import org.apache.hadoop.hive.metastore.model.MWMResourcePlan.Status;
-import org.apache.hadoop.hive.metastore.model.MWMTrigger;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
+import org.apache.hive.common.util.TxnIdUtils;
import org.apache.thrift.TException;
import org.datanucleus.AbstractNucleusContext;
import org.datanucleus.ClassLoaderResolver;
@@ -1303,10 +1192,16 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public void createTable(Table tbl) throws InvalidObjectException, MetaException {
boolean commited = false;
+ MTable mtbl = null;
+
try {
openTransaction();
- MTable mtbl = convertToMTable(tbl);
+ mtbl = convertToMTable(tbl);
+ if (TxnUtils.isTransactionalTable(tbl)) {
+ mtbl.setTxnId(tbl.getTxnId());
+ mtbl.setWriteIdList(tbl.getValidWriteIdList());
+ }
pm.makePersistent(mtbl);
if (tbl.getCreationMetadata() != null) {
@@ -1417,6 +1312,8 @@ public class ObjectStore implements RawStore, Configurable {
TableName.getQualified(catName, dbName, tableName));
}
+ Table table = convertToTable(tbl);
+
List<MConstraint> tabConstraints = listAllTableConstraintsWithOptionalConstraintName(
catName, dbName, tableName, null);
if (CollectionUtils.isNotEmpty(tabConstraints)) {
@@ -1515,17 +1412,51 @@ public class ObjectStore implements RawStore, Configurable {
return mConstraints;
}
+ private static String getFullyQualifiedTableName(String dbName, String tblName) {
+ return ((dbName == null || dbName.isEmpty()) ? "" : "\"" + dbName + "\".\"")
+ + "\"" + tblName + "\"";
+ }
+
+ @Override
+ public Table
+ getTable(String catName, String dbName, String tableName)
+ throws MetaException {
+ return getTable(catName, dbName, tableName, -1, null);
+ }
+
@Override
- public Table getTable(String catName, String dbName, String tableName) throws MetaException {
+ public Table getTable(String catName, String dbName, String tableName,
+ long txnId, String writeIdList)
+ throws MetaException {
boolean commited = false;
Table tbl = null;
try {
openTransaction();
- tbl = convertToTable(getMTable(catName, dbName, tableName));
+ MTable mtable = getMTable(catName, dbName, tableName);
+ tbl = convertToTable(mtable);
// Retrieve creation metadata if needed
if (tbl != null && TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) {
tbl.setCreationMetadata(
- convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName)));
+ convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName)));
+ }
+
+ // If transactional non partitioned table,
+ // check whether the current version table statistics
+ // in the metastore comply with the client query's snapshot isolation.
+ // Note: a partitioned table has table stats and table snapshot in MPartiiton.
+ if (writeIdList != null) {
+ if (tbl != null
+ && TxnUtils.isTransactionalTable(tbl)
+ && tbl.getPartitionKeysSize() == 0) {
+ if (isCurrentStatsValidForTheQuery(mtable, txnId, writeIdList, -1, false)) {
+ tbl.setIsStatsCompliant(IsolationLevelCompliance.YES);
+ } else {
+ tbl.setIsStatsCompliant(IsolationLevelCompliance.NO);
+ // Do not make persistent the following state since it is the query specific (not global).
+ StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from Table's parameters.");
+ }
+ }
}
commited = commitTransaction();
} finally {
@@ -2049,12 +1980,17 @@ public class ObjectStore implements RawStore, Configurable {
String ownerType = (ownerPrincipalType == null) ? PrincipalType.USER.name() : ownerPrincipalType.name();
// A new table is always created with a new column descriptor
- return new MTable(normalizeIdentifier(tbl.getTableName()), mdb,
+ MTable mtable = new MTable(normalizeIdentifier(tbl.getTableName()), mdb,
convertToMStorageDescriptor(tbl.getSd()), tbl.getOwner(), ownerType, tbl
.getCreateTime(), tbl.getLastAccessTime(), tbl.getRetention(),
convertToMFieldSchemas(tbl.getPartitionKeys()), tbl.getParameters(),
tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(),
tableType);
+ if (TxnUtils.isTransactionalTable(tbl)) {
+ mtable.setTxnId(tbl.getTxnId());
+ mtable.setWriteIdList(tbl.getValidWriteIdList());
+ }
+ return mtable;
}
private List<MFieldSchema> convertToMFieldSchemas(List<FieldSchema> keys) {
@@ -2331,6 +2267,7 @@ public class ObjectStore implements RawStore, Configurable {
+ dbName + "." + tblName + ": " + part);
}
MPartition mpart = convertToMPart(part, table, true);
+
toPersist.add(mpart);
int now = (int)(System.currentTimeMillis()/1000);
if (tabGrants != null) {
@@ -2442,6 +2379,7 @@ public class ObjectStore implements RawStore, Configurable {
MetaException {
boolean success = false;
boolean commited = false;
+
try {
String catName = part.isSetCatName() ? part.getCatName() : getDefaultCatalog(conf);
MTable table = this.getMTable(catName, part.getDbName(), part.getTableName());
@@ -2453,7 +2391,7 @@ public class ObjectStore implements RawStore, Configurable {
catName, part.getDbName(), part.getTableName());
}
openTransaction();
- MPartition mpart = convertToMPart(part, true);
+ MPartition mpart = convertToMPart(part, table, true);
pm.makePersistent(mpart);
int now = (int)(System.currentTimeMillis()/1000);
@@ -2495,14 +2433,38 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public Partition getPartition(String catName, String dbName, String tableName,
List<String> part_vals) throws NoSuchObjectException, MetaException {
+ return getPartition(catName, dbName, tableName, part_vals, -1, null);
+ }
+
+ @Override
+ public Partition getPartition(String catName, String dbName, String tableName,
+ List<String> part_vals,
+ long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException {
openTransaction();
- Partition part = convertToPart(getMPartition(catName, dbName, tableName, part_vals));
+ MTable table = this.getMTable(catName, dbName, tableName);
+ MPartition mpart = getMPartition(catName, dbName, tableName, part_vals);
+ Partition part = convertToPart(mpart);
commitTransaction();
if(part == null) {
throw new NoSuchObjectException("partition values="
+ part_vals.toString());
}
part.setValues(part_vals);
+ // If transactional table partition, check whether the current version partition
+ // statistics in the metastore comply with the client query's snapshot isolation.
+ if (writeIdList != null) {
+ if (TxnUtils.isTransactionalTable(table.getParameters())) {
+ if (isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) {
+ part.setIsStatsCompliant(IsolationLevelCompliance.YES);
+ } else {
+ part.setIsStatsCompliant(IsolationLevelCompliance.NO);
+ // Do not make persistent the following state since it is query specific (not global).
+ StatsSetupConst.setBasicStatsState(part.getParameters(), StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from Partition object's parameters.");
+ }
+ }
+ }
return part;
}
@@ -2601,26 +2563,6 @@ public class ObjectStore implements RawStore, Configurable {
* is true, then this partition's storage descriptor's column descriptor will point
* to the same one as the table's storage descriptor.
* @param part the partition to convert
- * @param useTableCD whether to try to use the parent table's column descriptor.
- * @return the model partition object, and null if the input partition is null.
- * @throws InvalidObjectException
- * @throws MetaException
- */
- private MPartition convertToMPart(Partition part, boolean useTableCD)
- throws InvalidObjectException, MetaException {
- if (part == null) {
- return null;
- }
- MTable mt = getMTable(part.getCatName(), part.getDbName(), part.getTableName());
- return convertToMPart(part, mt, useTableCD);
- }
-
- /**
- * Convert a Partition object into an MPartition, which is an object backed by the db
- * If the Partition's set of columns is the same as the parent table's AND useTableCD
- * is true, then this partition's storage descriptor's column descriptor will point
- * to the same one as the table's storage descriptor.
- * @param part the partition to convert
* @param mt the parent table object
* @param useTableCD whether to try to use the parent table's column descriptor.
* @return the model partition object, and null if the input partition is null.
@@ -2652,10 +2594,15 @@ public class ObjectStore implements RawStore, Configurable {
msd = convertToMStorageDescriptor(part.getSd());
}
- return new MPartition(Warehouse.makePartName(convertToFieldSchemas(mt
+ MPartition mpart = new MPartition(Warehouse.makePartName(convertToFieldSchemas(mt
.getPartitionKeys()), part.getValues()), mt, part.getValues(), part
.getCreateTime(), part.getLastAccessTime(),
msd, part.getParameters());
+ if (TxnUtils.isTransactionalTable(mt.getParameters())) {
+ mpart.setTxnId(part.getTxnId());
+ mpart.setWriteIdList(part.getValidWriteIdList());
+ }
+ return mpart;
}
private Partition convertToPart(MPartition mpart) throws MetaException {
@@ -3031,7 +2978,7 @@ public class ObjectStore implements RawStore, Configurable {
TableName.getQualified(catName, dbName, tableName), filter, cols);
List<String> partitionNames = null;
List<Partition> partitions = null;
- Table tbl = getTable(catName, dbName, tableName);
+ Table tbl = getTable(catName, dbName, tableName, -1, null);
try {
// Get partitions by name - ascending or descending
partitionNames = getPartitionNamesByFilter(catName, dbName, tableName, filter, ascending,
@@ -3164,7 +3111,8 @@ public class ObjectStore implements RawStore, Configurable {
if (applyDistinct) {
partValuesSelect.append("DISTINCT ");
}
- List<FieldSchema> partitionKeys = getTable(catName, dbName, tableName).getPartitionKeys();
+ List<FieldSchema> partitionKeys =
+ getTable(catName, dbName, tableName, -1, null).getPartitionKeys();
for (FieldSchema key : cols) {
partValuesSelect.append(extractPartitionKey(key, partitionKeys)).append(", ");
}
@@ -3246,7 +3194,7 @@ public class ObjectStore implements RawStore, Configurable {
catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
- Table table = getTable(catName, dbName, tableName);
+ Table table = getTable(catName, dbName, tableName, -1, null);
if (table == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tableName)
+ " table not found");
@@ -3622,7 +3570,8 @@ public class ObjectStore implements RawStore, Configurable {
protected T results = null;
public GetHelper(String catalogName, String dbName, String tblName,
- boolean allowSql, boolean allowJdo) throws MetaException {
+ boolean allowSql, boolean allowJdo)
+ throws MetaException {
assert allowSql || allowJdo;
this.allowJdo = allowJdo;
this.catName = (catalogName != null) ? normalizeIdentifier(catalogName) : null;
@@ -3840,7 +3789,7 @@ public class ObjectStore implements RawStore, Configurable {
private abstract class GetStatHelper extends GetHelper<ColumnStatistics> {
public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql,
- boolean allowJdo) throws MetaException {
+ boolean allowJdo, String writeIdList) throws MetaException {
super(catalogName, dbName, tblName, allowSql, allowJdo);
}
@@ -4140,6 +4089,21 @@ public class ObjectStore implements RawStore, Configurable {
oldt.setViewExpandedText(newt.getViewExpandedText());
oldt.setRewriteEnabled(newt.isRewriteEnabled());
+ // If transactional, update MTable to have txnId and the writeIdList
+ // for the current Stats updater query.
+ if (newTable.getValidWriteIdList() != null &&
+ TxnUtils.isTransactionalTable(newTable)) {
+ // Check concurrent INSERT case and set false to the flag.
+ if (isCurrentStatsValidForTheQuery(oldt, newt.getTxnId(), newt.getWriteIdList(),
+ -1, true)) {
+ StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " +
+ dbname + "." + name + ". will be made persistent.");
+ }
+ oldt.setTxnId(newTable.getTxnId());
+ oldt.setWriteIdList(newTable.getValidWriteIdList());
+ }
+
// commit the changes
success = commitTransaction();
} finally {
@@ -4192,8 +4156,9 @@ public class ObjectStore implements RawStore, Configurable {
catName = normalizeIdentifier(catName);
name = normalizeIdentifier(name);
dbname = normalizeIdentifier(dbname);
+ MTable table = this.getMTable(catName, dbname, name);
MPartition oldp = getMPartition(catName, dbname, name, part_vals);
- MPartition newp = convertToMPart(newPart, false);
+ MPartition newp = convertToMPart(newPart, table, false);
MColumnDescriptor oldCD = null;
MStorageDescriptor oldSD = oldp.getSd();
if (oldSD != null) {
@@ -4214,6 +4179,20 @@ public class ObjectStore implements RawStore, Configurable {
if (newp.getLastAccessTime() != oldp.getLastAccessTime()) {
oldp.setLastAccessTime(newp.getLastAccessTime());
}
+ // If transactional, add/update the MUPdaterTransaction
+ // for the current updater query.
+ if (newPart.getValidWriteIdList() != null &&
+ TxnUtils.isTransactionalTable(table.getParameters())) {
+ // Check concurrent INSERT case and set false to the flag.
+ if (!isCurrentStatsValidForTheQuery(oldp, newp.getTxnId(), newp.getWriteIdList(),
+ -1, true)) {
+ StatsSetupConst.setBasicStatsState(oldp.getParameters(), StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition " +
+ dbname + "." + name + "." + oldp.getPartitionName() + " will be made persistent.");
+ }
+ oldp.setTxnId(newPart.getTxnId());
+ oldp.setWriteIdList(newPart.getValidWriteIdList());
+ }
return oldCD;
}
@@ -4245,7 +4224,8 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public void alterPartitions(String catName, String dbname, String name,
- List<List<String>> part_vals, List<Partition> newParts)
+ List<List<String>> part_vals, List<Partition> newParts,
+ long txnId, String writeIdList)
throws InvalidObjectException, MetaException {
boolean success = false;
Exception e = null;
@@ -4255,6 +4235,10 @@ public class ObjectStore implements RawStore, Configurable {
Set<MColumnDescriptor> oldCds = new HashSet<>();
for (Partition tmpPart: newParts) {
List<String> tmpPartVals = part_val_itr.next();
+ if (txnId > 0) {
+ tmpPart.setTxnId(txnId);
+ tmpPart.setValidWriteIdList(writeIdList);
+ }
MColumnDescriptor oldCd = alterPartitionNoTxn(catName, dbname, name, tmpPartVals, tmpPart);
if (oldCd != null) {
oldCds.add(oldCd);
@@ -6131,7 +6115,9 @@ public class ObjectStore implements RawStore, Configurable {
} else if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
boolean found = false;
- Table tabObj = this.getTable(catName, hiveObject.getDbName(), hiveObject.getObjectName());
+ Table tabObj =
+ this.getTable(catName, hiveObject.getDbName(),
+ hiveObject.getObjectName(), -1, null);
String partName = null;
if (hiveObject.getPartValues() != null) {
partName = Warehouse.makePartName(tabObj.getPartitionKeys(), hiveObject.getPartValues());
@@ -6165,7 +6151,7 @@ public class ObjectStore implements RawStore, Configurable {
} else if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
Table tabObj = this.getTable(catName, hiveObject.getDbName(), hiveObject
- .getObjectName());
+ .getObjectName(), -1, null);
String partName = null;
if (hiveObject.getPartValues() != null) {
partName = Warehouse.makePartName(tabObj.getPartitionKeys(),
@@ -7687,7 +7673,7 @@ public class ObjectStore implements RawStore, Configurable {
query
.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4," +
"java.lang.String t5");
- Table tbl = getTable(catName, dbName, tblName); // Make sure dbName and tblName are valid.
+ Table tbl = getTable(catName, dbName, tblName, -1, null); // Make sure dbName and tblName are valid.
if (null == tbl) {
throw new UnknownTableException("Table: " + tblName + " is not found.");
}
@@ -7713,7 +7699,7 @@ public class ObjectStore implements RawStore, Configurable {
Table tbl = null;
try{
openTransaction();
- tbl = getTable(catName, dbName, tblName); // Make sure dbName and tblName are valid.
+ tbl = getTable(catName, dbName, tblName, -1, null); // Make sure dbName and tblName are valid.
if(null == tbl) {
throw new UnknownTableException("Table: "+ tblName + " is not found.");
}
@@ -8442,7 +8428,10 @@ public class ObjectStore implements RawStore, Configurable {
}
}
- private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, QueryWrapper queryWrapper)
+ private List<MTableColumnStatistics> getMTableColumnStatistics(
+ Table table,
+ List<String> colNames,
+ QueryWrapper queryWrapper)
throws MetaException {
if (colNames == null || colNames.isEmpty()) {
return Collections.emptyList();
@@ -8517,9 +8506,40 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName,
+ public ColumnStatistics getTableColumnStatistics(
+ String catName,
+ String dbName,
+ String tableName,
List<String> colNames) throws MetaException, NoSuchObjectException {
- return getTableColumnStatisticsInternal(catName, dbName, tableName, colNames, true, true);
+ return getTableColumnStatisticsInternal(
+ catName, dbName, tableName, colNames, true, true);
+ }
+
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ String catName,
+ String dbName,
+ String tableName,
+ List<String> colNames,
+ long txnId,
+ String writeIdList) throws MetaException, NoSuchObjectException {
+ IsolationLevelCompliance iLL = IsolationLevelCompliance.UNKNOWN;
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, set No to the compliance flag.
+ if (writeIdList != null) {
+ MTable table = this.getMTable(catName, dbName, tableName);
+ if (!isCurrentStatsValidForTheQuery(table, txnId, writeIdList, -1, false)) {
+ iLL = IsolationLevelCompliance.NO;
+ } else {
+ iLL = IsolationLevelCompliance.YES;
+ }
+ }
+ ColumnStatistics cS = getTableColumnStatisticsInternal(
+ catName, dbName, tableName, colNames, true, true);
+ if (cS != null) {
+ cS.setIsStatsCompliant(iLL);
+ }
+ return cS;
}
protected ColumnStatistics getTableColumnStatisticsInternal(
@@ -8527,7 +8547,7 @@ public class ObjectStore implements RawStore, Configurable {
boolean allowJdo) throws MetaException, NoSuchObjectException {
final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
return new GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName),
- normalizeIdentifier(tableName), allowSql, allowJdo) {
+ normalizeIdentifier(tableName), allowSql, allowJdo, null) {
@Override
protected ColumnStatistics getSqlResult(GetHelper<ColumnStatistics> ctx) throws MetaException {
return directSql.getTableStats(catName, dbName, tblName, colNames, enableBitVector);
@@ -8538,7 +8558,8 @@ public class ObjectStore implements RawStore, Configurable {
QueryWrapper queryWrapper = new QueryWrapper();
try {
- List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, queryWrapper);
+ List<MTableColumnStatistics> mStats =
+ getMTableColumnStatistics(getTable(), colNames, queryWrapper);
if (mStats.isEmpty()) {
return null;
}
@@ -8568,6 +8589,35 @@ public class ObjectStore implements RawStore, Configurable {
catName, dbName, tableName, partNames, colNames, true, true);
}
+ @Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ String catName, String dbName, String tableName,
+ List<String> partNames, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ // If any of the current partition stats in the metastore doesn't comply with
+ // the isolation level of the query, return null.
+ if (writeIdList != null) {
+ if (partNames == null && partNames.isEmpty()) {
+ LOG.warn("The given partNames does not have any name.");
+ return null;
+ }
+ // Loop through the given "partNames" list
+ // checking isolation-level-compliance of each partition column stats.
+ for(String partName : partNames) {
+ MPartition mpart = getMPartition(catName, dbName, tableName, Warehouse.getPartValuesFromPartName(partName));
+ if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) {
+ LOG.debug("The current metastore transactional partition column statistics " +
+ "for " + dbName + "." + tableName + "." + mpart.getPartitionName() + " is not valid " +
+ "for the current query.");
+ return null;
+ }
+ }
+ }
+ return getPartitionColumnStatisticsInternal(
+ catName, dbName, tableName, partNames, colNames, true, true);
+ }
+
protected List<ColumnStatistics> getPartitionColumnStatisticsInternal(
String catName, String dbName, String tableName, final List<String> partNames, final List<String> colNames,
boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
@@ -8616,10 +8666,36 @@ public class ObjectStore implements RawStore, Configurable {
}.run(true);
}
+ @Override
+ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName,
+ final List<String> partNames, final List<String> colNames,
+ long txnId, String writeIdList) throws MetaException, NoSuchObjectException {
+ // If the current stats in the metastore doesn't comply with
+ // the isolation level of the query, return null.
+ if (writeIdList != null) {
+ if (partNames == null && partNames.isEmpty()) {
+ LOG.warn("The given partNames does not have any name.");
+ return null;
+ }
+ // Loop through the given "partNames" list
+ // checking isolation-level-compliance of each partition column stats.
+ for(String partName : partNames) {
+ MPartition mpart = getMPartition(catName, dbName, tblName, Warehouse.getPartValuesFromPartName(partName));
+ if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) {
+ LOG.debug("The current metastore transactional partition column statistics " +
+ "for " + dbName + "." + tblName + "." + mpart.getPartitionName() + " is not valid " +
+ "for the current query.");
+ return null;
+ }
+ }
+ }
+ return get_aggr_stats_for(catName, dbName, tblName, partNames, colNames);
+ }
@Override
public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName,
- final List<String> partNames, final List<String> colNames) throws MetaException, NoSuchObjectException {
+ final List<String> partNames, final List<String> colNames)
+ throws MetaException, NoSuchObjectException {
final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(getConf(),
ConfVars.STATS_NDV_DENSITY_FUNCTION);
final double ndvTuner = MetastoreConf.getDoubleVar(getConf(), ConfVars.STATS_NDV_TUNER);
@@ -8651,7 +8727,8 @@ public class ObjectStore implements RawStore, Configurable {
throws MetaException, NoSuchObjectException {
final boolean enableBitVector =
MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
- return new GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>>(catName, dbName, null, true, false) {
+ return new GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>>(
+ catName, dbName, null, true, false) {
@Override
protected List<MetaStoreUtils.ColStatsObjWithSourceInfo> getSqlResult(
GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>> ctx) throws MetaException {
@@ -12109,4 +12186,88 @@ public class ObjectStore implements RawStore, Configurable {
return ret;
}
+ /**
+ * Return true if the current statistics in the Metastore is valid
+ * for the query of the given "txnId" and "queryValidWriteIdList".
+ *
+ * Note that a statistics entity is valid iff
+ * the stats is written by the current query or
+ * the conjunction of the following two are true:
+ * ~ COLUMN_STATE_ACCURATE(CSA) state is true
+ * ~ Isolation-level (snapshot) compliant with the query
+ * @param tbl MTable of the stats entity
+ * @param txnId transaction id of the query
+ * @param queryValidWriteIdList valid writeId list of the query
+ * @Precondition "tbl" should be retrieved from the TBLS table.
+ */
+ private boolean isCurrentStatsValidForTheQuery(
+ MTable tbl, long txnId, String queryValidWriteIdList,
+ long statsWriteId, boolean checkConcurrentWrites)
+ throws MetaException {
+ return isCurrentStatsValidForTheQuery(tbl.getTxnId(), tbl.getParameters(), tbl.getWriteIdList(),
+ txnId, queryValidWriteIdList, statsWriteId, checkConcurrentWrites);
+ }
+
+ /**
+ * Return true if the current statistics in the Metastore is valid
+ * for the query of the given "txnId" and "queryValidWriteIdList".
+ *
+ * Note that a statistics entity is valid iff
+ * the stats is written by the current query or
+ * the conjunction of the following two are true:
+ * ~ COLUMN_STATE_ACCURATE(CSA) state is true
+ * ~ Isolation-level (snapshot) compliant with the query
+ * @param part MPartition of the stats entity
+ * @param txnId transaction id of the query
+ * @param queryValidWriteIdList valid writeId list of the query
+ * @Precondition "part" should be retrieved from the PARTITIONS table.
+ */
+ private boolean isCurrentStatsValidForTheQuery(
+ MPartition part, long txnId, String queryValidWriteIdList,
+ long statsWriteId, boolean checkConcurrentWrites)
+ throws MetaException {
+ return isCurrentStatsValidForTheQuery(part.getTxnId(), part.getParameters(), part.getWriteIdList(),
+ txnId, queryValidWriteIdList, statsWriteId, checkConcurrentWrites);
+ }
+
+ private boolean isCurrentStatsValidForTheQuery(
+ long statsTxnId, Map<String, String> statsParams, String statsWriteIdList,
+ long queryTxnId, String queryValidWriteIdList,
+ long statsWriteId, boolean checkConcurrentWrites)
+ throws MetaException {
+ // If the current query is a stats updater, then we can return true
+ // to avoid implementing a logic inside TxnIdUtils.checkEquivalentWriteIds().
+ if (statsTxnId == queryTxnId) {
+ return true;
+ }
+
+ // If the Metastore stats's writer transaction is open or aborted
+ // we should return false.
+ try {
+ if (TxnDbUtil.isOpenOrAbortedTransaction(conf, statsTxnId)) {
+ return false;
+ }
+ } catch (Exception e) {
+ throw new MetaException("Cannot check transaction state.");
+ }
+
+ // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is
+ // written by an aborted transaction but TXNS has no entry for the transaction
+ // after compaction.
+ if (!StatsSetupConst.areBasicStatsUptoDate(statsParams)) {
+ return false;
+ }
+
+ // If the NUM_FILES of the table/partition is 0, return 'true' from this method.
+ // Since newly initialized empty table has 0 for the parameter.
+ if (Long.parseLong(statsParams.get(StatsSetupConst.NUM_FILES)) == 0) {
+ return true;
+ }
+
+ ValidWriteIdList list4Stats = new ValidReaderWriteIdList(statsWriteIdList);
+ ValidWriteIdList list4TheQuery = new ValidReaderWriteIdList(queryValidWriteIdList);
+
+ return !checkConcurrentWrites ? TxnIdUtils.checkEquivalentWriteIds(list4Stats, list4TheQuery) :
+ !TxnIdUtils.areTheseConcurrentWrites(list4Stats, list4TheQuery, statsWriteId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index bbbdf21..e1c1ab9 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -19,10 +19,7 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
+import org.apache.hadoop.hive.metastore.api.*;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
@@ -34,59 +31,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.RuntimeStat;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.WMMapping;
-import org.apache.hadoop.hive.metastore.api.WMNullablePool;
-import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMPool;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMTrigger;
-import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
import org.apache.thrift.TException;
@@ -266,6 +210,20 @@ public interface RawStore extends Configurable {
Table getTable(String catalogName, String dbName, String tableName) throws MetaException;
/**
+ * Get a table object.
+ * @param catalogName catalog the table is in.
+ * @param dbName database the table is in.
+ * @param tableName table name.
+ * @param txnId transaction id of the calling transaction
+ * @param writeIdList string format of valid writeId transaction list
+ * @return table object, or null if no such table exists (wow it would be nice if we either
+ * consistently returned null or consistently threw NoSuchObjectException).
+ * @throws MetaException something went wrong in the RDBMS
+ */
+ Table getTable(String catalogName, String dbName, String tableName,
+ long txnId, String writeIdList) throws MetaException;
+
+ /**
* Add a partition.
* @param part partition to add
* @return true if the partition was successfully added.
@@ -317,6 +275,22 @@ public interface RawStore extends Configurable {
*/
Partition getPartition(String catName, String dbName, String tableName,
List<String> part_vals) throws MetaException, NoSuchObjectException;
+ /**
+ * Get a partition.
+ * @param catName catalog name.
+ * @param dbName database name.
+ * @param tableName table name.
+ * @param part_vals partition values for this table.
+ * @param txnId transaction id of the calling transaction
+ * @param writeIdList string format of valid writeId transaction list
+ * @return the partition.
+ * @throws MetaException error reading from RDBMS.
+ * @throws NoSuchObjectException no partition matching this specification exists.
+ */
+ Partition getPartition(String catName, String dbName, String tableName,
+ List<String> part_vals,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException;
/**
* Check whether a partition exists.
@@ -525,11 +499,14 @@ public interface RawStore extends Configurable {
* @param new_parts list of new partitions. The order must match the old partitions described in
* part_vals_list. Each of these should be a complete copy of the new
* partition, not just the pieces to update.
+ * @param txnId transaction id of the transaction that called this method.
+ * @param writeIdList valid write id list of the transaction on the current table
* @throws InvalidObjectException One of the indicated partitions does not exist.
* @throws MetaException error accessing the RDBMS.
*/
void alterPartitions(String catName, String db_name, String tbl_name,
- List<List<String>> part_vals_list, List<Partition> new_parts)
+ List<List<String>> part_vals_list, List<Partition> new_parts,
+ long txnId, String writeIdList)
throws InvalidObjectException, MetaException;
/**
@@ -901,6 +878,25 @@ public interface RawStore extends Configurable {
List<String> colName) throws MetaException, NoSuchObjectException;
/**
+ * Returns the relevant column statistics for a given column in a given table in a given database
+ * if such statistics exist.
+ * @param catName catalog name.
+ * @param dbName name of the database, defaults to current database
+ * @param tableName name of the table
+ * @param colName names of the columns for which statistics is requested
+ * @param txnId transaction id of the calling transaction
+ * @param writeIdList string format of valid writeId transaction list
+ * @return Relevant column statistics for the column for the given table
+ * @throws NoSuchObjectException No such table
+ * @throws MetaException error accessing the RDBMS
+ *
+ */
+ ColumnStatistics getTableColumnStatistics(
+ String catName, String dbName, String tableName,
+ List<String> colName, long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException;
+
+ /**
* Get statistics for a partition for a set of columns.
* @param catName catalog name.
* @param dbName database name.
@@ -916,6 +912,25 @@ public interface RawStore extends Configurable {
throws MetaException, NoSuchObjectException;
/**
+ * Get statistics for a partition for a set of columns.
+ * @param catName catalog name.
+ * @param dbName database name.
+ * @param tblName table name.
+ * @param partNames list of partition names. These are names so must be key1=val1[/key2=val2...]
+ * @param colNames list of columns to get stats for
+ * @param txnId transaction id of the calling transaction
+ * @param writeIdList string format of valid writeId transaction list
+ * @return list of statistics objects
+ * @throws MetaException error accessing the RDBMS
+ * @throws NoSuchObjectException no such partition.
+ */
+ List<ColumnStatistics> getPartitionColumnStatistics(
+ String catName, String dbName, String tblName,
+ List<String> partNames, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException;
+
+ /**
* Deletes column statistics if present associated with a given db, table, partition and col. If
* null is passed instead of a colName, stats when present for all columns associated
* with a given db, table and partition are deleted.
@@ -1159,6 +1174,25 @@ public interface RawStore extends Configurable {
List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException;
/**
+ * Get aggregated stats for a table or partition(s).
+ * @param catName catalog name.
+ * @param dbName database name.
+ * @param tblName table name.
+ * @param partNames list of partition names. These are the names of the partitions, not
+ * values.
+ * @param colNames list of column names
+ * @param txnId transaction id of the calling transaction
+ * @param writeIdList string format of valid writeId transaction list
+ * @return aggregated stats
+ * @throws MetaException error accessing RDBMS
+ * @throws NoSuchObjectException no such table or partition
+ */
+ AggrStats get_aggr_stats_for(String catName, String dbName, String tblName,
+ List<String> partNames, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException;
+
+ /**
* Get column stats for all partitions of all tables in the database
* @param catName catalog name
* @param dbName database name
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 7c3588d..ad05051 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -49,68 +49,10 @@ import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
import org.apache.hadoop.hive.metastore.RawStore;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.WMNullablePool;
-import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMTrigger;
-import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.cache.SharedCache.StatsType;
import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator;
import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.RuntimeStat;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMMapping;
-import org.apache.hadoop.hive.metastore.api.WMPool;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
@@ -879,20 +821,29 @@ public class CachedStore implements RawStore, Configurable {
@Override
public Table getTable(String catName, String dbName, String tblName) throws MetaException {
+ return getTable(catName, dbName, tblName, -1, null);
+ }
+
+ // TODO: if writeIdList is not null, check isolation level compliance for SVS,
+ // possibly with getTableFromCache() with table snapshot in cache.
+ @Override
+ public Table getTable(String catName, String dbName, String tblName,
+ long txnId, String writeIdList)
+ throws MetaException {
catName = normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- return rawStore.getTable(catName, dbName, tblName);
+ return rawStore.getTable(catName, dbName, tblName, txnId,writeIdList);
}
Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName);
- if (tbl == null) {
+ if (tbl == null || writeIdList != null) {
// This table is not yet loaded in cache
// If the prewarm thread is working on this table's database,
// let's move this table to the top of tblNamesBeingPrewarmed stack,
// so that it gets loaded to the cache faster and is available for subsequent requests
tblsPendingPrewarm.prioritizeTableForPrewarm(tblName);
- return rawStore.getTable(catName, dbName, tblName);
+ return rawStore.getTable(catName, dbName, tblName, txnId, writeIdList);
}
if (tbl != null) {
tbl.unsetPrivileges();
@@ -955,16 +906,26 @@ public class CachedStore implements RawStore, Configurable {
@Override
public Partition getPartition(String catName, String dbName, String tblName, List<String> part_vals)
throws MetaException, NoSuchObjectException {
+ return getPartition(catName, dbName, tblName, part_vals, -1, null);
+ }
+
+ // TODO: the same as getTable()
+ @Override
+ public Partition getPartition(String catName, String dbName, String tblName,
+ List<String> part_vals, long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
catName = normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- return rawStore.getPartition(catName, dbName, tblName, part_vals);
+ return rawStore.getPartition(
+ catName, dbName, tblName, part_vals, txnId, writeIdList);
}
Partition part = sharedCache.getPartitionFromCache(catName, dbName, tblName, part_vals);
- if (part == null) {
+ if (part == null || writeIdList != null) {
// The table containing the partition is not yet loaded in cache
- return rawStore.getPartition(catName, dbName, tblName, part_vals);
+ return rawStore.getPartition(
+ catName, dbName, tblName, part_vals, txnId, writeIdList);
}
return part;
}
@@ -1204,15 +1165,17 @@ public class CachedStore implements RawStore, Configurable {
@Override
public void alterPartitions(String catName, String dbName, String tblName,
- List<List<String>> partValsList, List<Partition> newParts)
+ List<List<String>> partValsList, List<Partition> newParts,
+ long txnId, String writeIdList)
throws InvalidObjectException, MetaException {
- rawStore.alterPartitions(catName, dbName, tblName, partValsList, newParts);
+ rawStore.alterPartitions(catName, dbName, tblName, partValsList, newParts, txnId, writeIdList);
catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tblName = normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
return;
}
+ // TODO: modify the following method for the case when writeIdList != null.
sharedCache.alterPartitionsInCache(catName, dbName, tblName, partValsList, newParts);
}
@@ -1656,16 +1619,27 @@ public class CachedStore implements RawStore, Configurable {
@Override
public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tblName,
List<String> colNames) throws MetaException, NoSuchObjectException {
+ return getTableColumnStatistics(catName, dbName, tblName, colNames, -1, null);
+ }
+
+ // TODO: the same as getTable()
+ @Override
+ public ColumnStatistics getTableColumnStatistics(
+ String catName, String dbName, String tblName, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
catName = StringUtils.normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- return rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames);
+ return rawStore.getTableColumnStatistics(
+ catName, dbName, tblName, colNames, txnId, writeIdList);
}
Table table = sharedCache.getTableFromCache(catName, dbName, tblName);
- if (table == null) {
+ if (table == null || writeIdList != null) {
// The table is not yet loaded in cache
- return rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames);
+ return rawStore.getTableColumnStatistics(
+ catName, dbName, tblName, colNames, txnId, writeIdList);
}
ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tblName);
List<ColumnStatisticsObj> colStatObjs =
@@ -1723,6 +1697,15 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ String catName, String dbName, String tblName, List<String> partNames,
+ List<String> colNames, long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return rawStore.getPartitionColumnStatistics(
+ catName, dbName, tblName, partNames, colNames, txnId, writeIdList);
+ }
+
+ @Override
public boolean deletePartitionColumnStatistics(String catName, String dbName, String tblName, String partName,
List<String> partVals, String colName)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
@@ -1743,17 +1726,28 @@ public class CachedStore implements RawStore, Configurable {
@Override
public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, List<String> partNames,
List<String> colNames) throws MetaException, NoSuchObjectException {
+ return get_aggr_stats_for(catName, dbName, tblName, partNames, colNames, -1, null);
+ }
+
+ @Override
+ // TODO: the same as getTable() for transactional stats.
+ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName,
+ List<String> partNames, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
List<ColumnStatisticsObj> colStats;
catName = normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- rawStore.get_aggr_stats_for(catName, dbName, tblName, partNames, colNames);
+ rawStore.get_aggr_stats_for(
+ catName, dbName, tblName, partNames, colNames, txnId, writeIdList);
}
Table table = sharedCache.getTableFromCache(catName, dbName, tblName);
- if (table == null) {
+ if (table == null || writeIdList != null) {
// The table is not yet loaded in cache
- return rawStore.get_aggr_stats_for(catName, dbName, tblName, partNames, colNames);
+ return rawStore.get_aggr_stats_for(
+ catName, dbName, tblName, partNames, colNames, txnId, writeIdList);
}
List<String> allPartNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1);
if (partNames.size() == allPartNames.size()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java
index 4a97f89..56f9048 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java
@@ -30,7 +30,8 @@ public class MPartition {
private int lastAccessTime;
private MStorageDescriptor sd;
private Map<String, String> parameters;
-
+ private long txnId;
+ private String writeIdList;
public MPartition() {}
@@ -152,4 +153,19 @@ public class MPartition {
this.createTime = createTime;
}
+ public long getTxnId() {
+ return txnId;
+ }
+
+ public void setTxnId(long txnId) {
+ this.txnId = txnId;
+ }
+
+ public String getWriteIdList() {
+ return writeIdList;
+ }
+
+ public void setWriteIdList(String writeIdList) {
+ this.writeIdList = writeIdList;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
index 50d9c5b..ff68eba 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
@@ -55,6 +55,7 @@ public class MPartitionColumnStatistics {
private Long numTrues;
private Long numFalses;
private long lastAnalyzed;
+ private long txnId;
public MPartitionColumnStatistics() {}
@@ -278,4 +279,12 @@ public class MPartitionColumnStatistics {
public void setBitVector(byte[] bitVector) {
this.bitVector = bitVector;
}
+
+ public long getTxnId() {
+ return txnId;
+ }
+
+ public void setTxnId(long txnId) {
+ this.txnId = txnId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
index 38ad479..7ef1ef6 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
@@ -1,3 +1,4 @@
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -37,6 +38,8 @@ public class MTable {
private String viewExpandedText;
private boolean rewriteEnabled;
private String tableType;
+ private long txnId;
+ private String writeIdList;
public MTable() {}
@@ -270,4 +273,20 @@ public class MTable {
public String getTableType() {
return tableType;
}
+
+ public long getTxnId() {
+ return txnId;
+ }
+
+ public void setTxnId(long txnId) {
+ this.txnId = txnId;
+ }
+
+ public String getWriteIdList() {
+ return writeIdList;
+ }
+
+ public void setWriteIdList(String writeIdList) {
+ this.writeIdList = writeIdList;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
index 731cd6f..9d687e4 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
@@ -53,6 +53,7 @@ public class MTableColumnStatistics {
private Long numTrues;
private Long numFalses;
private long lastAnalyzed;
+ private long txnId;
public MTableColumnStatistics() {}
@@ -269,4 +270,12 @@ public class MTableColumnStatistics {
public void setBitVector(byte[] bitVector) {
this.bitVector = bitVector;
}
+
+ public long getTxnId() {
+ return txnId;
+ }
+
+ public void setTxnId(long txnId) {
+ this.txnId = txnId;
+ }
}