You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/08/31 00:14:40 UTC
svn commit: r1621542 - in /hive/trunk/metastore/src:
java/org/apache/hadoop/hive/metastore/ test/org/apache/hadoop/hive/metastore/
Author: hashutosh
Date: Sat Aug 30 22:14:40 2014
New Revision: 1621542
URL: http://svn.apache.org/r1621542
Log:
HIVE-7876 : further improve the columns stats update speed for all the partitions of a table (Pengcheng Xiong via Ashutosh Chauhan)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1621542&r1=1621541&r2=1621542&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Sat Aug 30 22:14:40 2014
@@ -3719,6 +3719,19 @@ public class HiveMetaStore extends Thrif
} finally {
endFunction("write_partition_column_statistics: ", ret != false, null, tableName);
}
+ }
+ public boolean update_partition_column_statistics(
+ SetPartitionsStatsRequest request) throws NoSuchObjectException,
+ InvalidObjectException, MetaException, TException,
+ InvalidInputException {
+ boolean ret = false;
+ try {
+ ret = getMS().updatePartitionColumnStatistics(request);
+ return ret;
+ } finally {
+ endFunction("write_partition_column_statistics: ", ret != false, null,
+ null);
+ }
}
@Override
@@ -5046,15 +5059,11 @@ public class HiveMetaStore extends Thrif
public boolean set_aggr_stats_for(SetPartitionsStatsRequest request)
throws NoSuchObjectException, InvalidObjectException, MetaException,
InvalidInputException, TException {
- boolean ret = true;
- for (ColumnStatistics colStats : request.getColStats()) {
- ret = ret && update_partition_column_statistics(colStats);
- }
- return ret;
+ return update_partition_column_statistics(request);
}
-
}
+
public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf) throws MetaException {
return newHMSHandler(name, hiveConf, false);
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1621542&r1=1621541&r2=1621542&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Sat Aug 30 22:14:40 2014
@@ -89,6 +89,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -5716,7 +5717,7 @@ public class ObjectStore implements RawS
pm.makePersistent(mStatsObj);
}
}
-
+
@Override
public boolean updateTableColumnStatistics(ColumnStatistics colStats)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
@@ -5777,6 +5778,34 @@ public class ObjectStore implements RawS
}
}
+ @Override
+ public boolean updatePartitionColumnStatistics(SetPartitionsStatsRequest request)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ boolean committed = false;
+ try {
+ openTransaction();
+ for (ColumnStatistics colStats : request.getColStats()) {
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ statsDesc.setDbName(statsDesc.getDbName().toLowerCase());
+ statsDesc.setTableName(statsDesc.getTableName().toLowerCase());
+ List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ statsObj.setColName(statsObj.getColName().toLowerCase());
+ statsObj.setColType(statsObj.getColType().toLowerCase());
+ MPartitionColumnStatistics mStatsObj = StatObjectConverter
+ .convertToMPartitionColumnStatistics(null, statsDesc, statsObj);
+ pm.makePersistent(mStatsObj);
+ }
+ }
+ committed = commitTransaction();
+ return committed;
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ }
+
private List<MTableColumnStatistics> getMTableColumnStatistics(
Table table, List<String> colNames) throws MetaException {
boolean committed = false;
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1621542&r1=1621541&r2=1621542&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Sat Aug 30 22:14:40 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -551,4 +552,8 @@ public interface RawStore extends Config
public AggrStats get_aggr_stats_for(String dbName, String tblName,
List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException;
+
+ boolean updatePartitionColumnStatistics(
+ SetPartitionsStatsRequest request) throws NoSuchObjectException,
+ MetaException, InvalidObjectException, InvalidInputException;
}
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1621542&r1=1621541&r2=1621542&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Sat Aug 30 22:14:40 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -719,5 +720,10 @@ public class DummyRawStoreControlledComm
return null;
}
+ @Override
+ public boolean updatePartitionColumnStatistics(SetPartitionsStatsRequest request)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return objectStore.updatePartitionColumnStatistics(request);
+ }
}
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1621542&r1=1621541&r2=1621542&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Sat Aug 30 22:14:40 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -735,6 +736,12 @@ public class DummyRawStoreForJdoConnecti
throws MetaException {
return null;
}
+
+ @Override
+ public boolean updatePartitionColumnStatistics(SetPartitionsStatsRequest request)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return false;
+ }
}