You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/04/30 20:29:07 UTC
hive git commit: HIVE-19319 : RuntimeStats fixes (Zoltan Haindrich
via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 95d6d9b3d -> 577d32ef7
HIVE-19319 : RuntimeStats fixes (Zoltan Haindrich via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/577d32ef
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/577d32ef
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/577d32ef
Branch: refs/heads/master
Commit: 577d32ef7d4ab0bd464d3b42426bb5a974655616
Parents: 95d6d9b
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Mon Apr 30 13:24:55 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Apr 30 13:28:42 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +-
.../listener/DummyRawStoreFailEvent.java | 8 +-
.../hive/ql/plan/mapper/CachingStatsSource.java | 2 +-
.../ql/plan/mapper/MetastoreStatsConnector.java | 42 +++-
.../hive/ql/plan/mapper/StatsSources.java | 7 +-
.../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 8 +-
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 57 +++++-
.../gen/thrift/gen-cpp/hive_metastore_types.h | 14 +-
.../metastore/api/GetRuntimeStatsRequest.java | 201 ++++++++++++++++++-
.../metastore/api/hive_metastoreConstants.java | 7 +-
.../src/gen/thrift/gen-php/metastore/Types.php | 50 ++++-
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 36 ++++
.../gen/thrift/gen-rb/hive_metastore_types.rb | 7 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../hive/metastore/HiveMetaStoreClient.java | 8 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 3 +-
.../hadoop/hive/metastore/ObjectStore.java | 75 ++++---
.../apache/hadoop/hive/metastore/RawStore.java | 5 +-
.../hive/metastore/RuntimeStatsCleanerTask.java | 3 +-
.../hive/metastore/cache/CachedStore.java | 8 +-
.../hive/metastore/conf/MetastoreConf.java | 3 -
.../src/main/thrift/hive_metastore.thrift | 2 +
.../DummyRawStoreControlledCommit.java | 8 +-
.../DummyRawStoreForJdoConnection.java | 4 +-
.../HiveMetaStoreClientPreCatalog.java | 3 +-
.../hive/metastore/client/TestRuntimeStats.java | 58 +++++-
26 files changed, 533 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6e35653..09bc572 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4276,12 +4276,15 @@ public class HiveConf extends Configuration {
+ " hiveserver: runtime statistics are persisted in the hiveserver - all sessions share it\n"
+ " metastore: runtime statistics are persisted in the metastore as well"),
+
HIVE_QUERY_MAX_REEXECUTION_COUNT("hive.query.reexecution.max.count", 1,
"Maximum number of re-executions for a single query."),
HIVE_QUERY_REEXECUTION_ALWAYS_COLLECT_OPERATOR_STATS("hive.query.reexecution.always.collect.operator.stats", false,
"If sessionstats are enabled; this option can be used to collect statistics all the time"),
+ HIVE_QUERY_REEXECUTION_STATS_CACHE_BATCH_SIZE("hive.query.reexecution.stats.cache.batch.size", -1,
+ "If runtime stats are stored in metastore; the maximal batch size per round during load."),
HIVE_QUERY_REEXECUTION_STATS_CACHE_SIZE("hive.query.reexecution.stats.cache.size", 100_000,
- "Size of the runtime statistics cache. Unit is: OperatorStat entry; a query plan consist ~100. See also: runtime.stats.max.entries"),
+ "Size of the runtime statistics cache. Unit is: OperatorStat entry; a query plan consist ~100."),
HIVE_QUERY_RESULTS_CACHE_ENABLED("hive.query.results.cache.enabled", true,
"If the query results cache is enabled. This will keep results of previously executed queries " +
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 8ecbaad..2e96983 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -1205,13 +1205,13 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws MetaException {
- return objectStore.getRuntimeStats();
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
+ return objectStore.getRuntimeStats(maxEntries, maxCreateTime);
}
@Override
- public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) {
- return 0;
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
+ return objectStore.deleteRuntimeStats(maxRetainSecs);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java
index 2841638..6b440e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java
@@ -49,7 +49,7 @@ public class CachingStatsSource implements StatsSource {
@Override
public boolean canProvideStatsFor(Class<?> clazz) {
- if (Operator.class.isAssignableFrom(clazz)) {
+ if (cache.size() > 0 && Operator.class.isAssignableFrom(clazz)) {
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java
index 237c1cc..f9ee46d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java
@@ -50,7 +50,7 @@ class MetastoreStatsConnector implements StatsSource {
private ExecutorService executor;
- MetastoreStatsConnector(StatsSource ss) {
+ MetastoreStatsConnector(int cacheSize, int batchSize, StatsSource ss) {
this.ss = ss;
executor = Executors.newSingleThreadExecutor(
new BasicThreadFactory.Builder()
@@ -58,22 +58,43 @@ class MetastoreStatsConnector implements StatsSource {
.daemon(true)
.build());
- executor.submit(new RuntimeStatsLoader());
+ executor.submit(new RuntimeStatsLoader(cacheSize, batchSize));
}
private class RuntimeStatsLoader implements Runnable {
+ private int maxEntriesToLoad;
+ private int batchSize;
+
+ public RuntimeStatsLoader(int maxEntriesToLoad, int batchSize) {
+ this.maxEntriesToLoad = maxEntriesToLoad;
+ if (batchSize <= 0) {
+ this.batchSize = -1;
+ } else {
+ this.batchSize = batchSize;
+ }
+ }
+
@Override
public void run() {
+ int lastCreateTime = Integer.MAX_VALUE;
+ int loadedEntries = 0;
try {
- List<RuntimeStat> rs = Hive.get().getMSC().getRuntimeStats();
- for (RuntimeStat thriftStat : rs) {
- try {
- ss.putAll(decode(thriftStat));
- } catch (IOException e) {
- logException("Exception while loading runtime stats", e);
+ do {
+ List<RuntimeStat> rs = Hive.get().getMSC().getRuntimeStats(batchSize, lastCreateTime);
+ if (rs.size() == 0) {
+ break;
+ }
+ for (RuntimeStat thriftStat : rs) {
+ loadedEntries += thriftStat.getWeight();
+ lastCreateTime = Math.min(lastCreateTime, thriftStat.getCreateTime() - 1);
+ try {
+ ss.putAll(decode(thriftStat));
+ } catch (IOException e) {
+ logException("Exception while loading runtime stats", e);
+ }
}
- }
+ } while (batchSize > 0 && loadedEntries < maxEntriesToLoad);
} catch (TException | HiveException e) {
logException("Exception while reading metastore runtime stats", e);
}
@@ -92,6 +113,9 @@ class MetastoreStatsConnector implements StatsSource {
@Override
public void putAll(Map<OpTreeSignature, OperatorStats> map) {
+ if (map.size() == 0) {
+ return;
+ }
ss.putAll(map);
executor.submit(new RuntimeStatsSubmitter(map));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
index 30b6a30..5a62046 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java
@@ -50,13 +50,14 @@ public class StatsSources {
public static StatsSource getStatsSource(HiveConf conf) {
String mode = conf.getVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_PERSISTENCE);
int cacheSize = conf.getIntVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_CACHE_SIZE);
+ int batchSize = conf.getIntVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_CACHE_BATCH_SIZE);
switch (mode) {
case "query":
return new MapBackedStatsSource();
case "hiveserver":
return StatsSources.globalStatsSource(cacheSize);
case "metastore":
- return StatsSources.metastoreBackedStatsSource(StatsSources.globalStatsSource(cacheSize));
+ return StatsSources.metastoreBackedStatsSource(cacheSize, batchSize, StatsSources.globalStatsSource(cacheSize));
default:
throw new RuntimeException("Unknown StatsSource setting: " + mode);
}
@@ -111,9 +112,9 @@ public class StatsSources {
return globalStatsSource;
}
- public static StatsSource metastoreBackedStatsSource(StatsSource parent) {
+ public static StatsSource metastoreBackedStatsSource(int cacheSize, int batchSize, StatsSource parent) {
if (metastoreStatsConnector == null) {
- metastoreStatsConnector = new MetastoreStatsConnector(parent);
+ metastoreStatsConnector = new MetastoreStatsConnector(cacheSize, batchSize, parent);
}
return metastoreStatsConnector;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java
index 8dc7387..09045af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.reexec;
import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -30,7 +29,6 @@ import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
-import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
import org.apache.hadoop.hive.ql.plan.mapper.StatsSources;
import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook;
import org.slf4j.Logger;
@@ -96,8 +94,10 @@ public class ReOptimizePlugin implements IReExecutionPlugin {
public void prepareToReExecute() {
statsReaderHook.setCollectOnSuccess(true);
retryPossible = false;
- coreDriver.setStatsSource(
- StatsSources.getStatsSourceContaining(coreDriver.getStatsSource(), coreDriver.getPlanMapper()));
+ if (!alwaysCollectStats) {
+ coreDriver.setStatsSource(
+ StatsSources.getStatsSourceContaining(coreDriver.getStatsSource(), coreDriver.getPlanMapper()));
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 2fab857..c6fadf8 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -30689,6 +30689,14 @@ GetRuntimeStatsRequest::~GetRuntimeStatsRequest() throw() {
}
+void GetRuntimeStatsRequest::__set_maxWeight(const int32_t val) {
+ this->maxWeight = val;
+}
+
+void GetRuntimeStatsRequest::__set_maxCreateTime(const int32_t val) {
+ this->maxCreateTime = val;
+}
+
uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
@@ -30701,6 +30709,8 @@ uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* ipr
using ::apache::thrift::protocol::TProtocolException;
+ bool isset_maxWeight = false;
+ bool isset_maxCreateTime = false;
while (true)
{
@@ -30708,12 +30718,37 @@ uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_STOP) {
break;
}
- xfer += iprot->skip(ftype);
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->maxWeight);
+ isset_maxWeight = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->maxCreateTime);
+ isset_maxCreateTime = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
xfer += iprot->readFieldEnd();
}
xfer += iprot->readStructEnd();
+ if (!isset_maxWeight)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_maxCreateTime)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
return xfer;
}
@@ -30722,6 +30757,14 @@ uint32_t GetRuntimeStatsRequest::write(::apache::thrift::protocol::TProtocol* op
apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
xfer += oprot->writeStructBegin("GetRuntimeStatsRequest");
+ xfer += oprot->writeFieldBegin("maxWeight", ::apache::thrift::protocol::T_I32, 1);
+ xfer += oprot->writeI32(this->maxWeight);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("maxCreateTime", ::apache::thrift::protocol::T_I32, 2);
+ xfer += oprot->writeI32(this->maxCreateTime);
+ xfer += oprot->writeFieldEnd();
+
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -30729,20 +30772,24 @@ uint32_t GetRuntimeStatsRequest::write(::apache::thrift::protocol::TProtocol* op
void swap(GetRuntimeStatsRequest &a, GetRuntimeStatsRequest &b) {
using ::std::swap;
- (void) a;
- (void) b;
+ swap(a.maxWeight, b.maxWeight);
+ swap(a.maxCreateTime, b.maxCreateTime);
}
GetRuntimeStatsRequest::GetRuntimeStatsRequest(const GetRuntimeStatsRequest& other1159) {
- (void) other1159;
+ maxWeight = other1159.maxWeight;
+ maxCreateTime = other1159.maxCreateTime;
}
GetRuntimeStatsRequest& GetRuntimeStatsRequest::operator=(const GetRuntimeStatsRequest& other1160) {
- (void) other1160;
+ maxWeight = other1160.maxWeight;
+ maxCreateTime = other1160.maxCreateTime;
return *this;
}
void GetRuntimeStatsRequest::printTo(std::ostream& out) const {
using ::apache::thrift::to_string;
out << "GetRuntimeStatsRequest(";
+ out << "maxWeight=" << to_string(maxWeight);
+ out << ", " << "maxCreateTime=" << to_string(maxCreateTime);
out << ")";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index cd78f58..6298893 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -13002,13 +13002,23 @@ class GetRuntimeStatsRequest {
GetRuntimeStatsRequest(const GetRuntimeStatsRequest&);
GetRuntimeStatsRequest& operator=(const GetRuntimeStatsRequest&);
- GetRuntimeStatsRequest() {
+ GetRuntimeStatsRequest() : maxWeight(0), maxCreateTime(0) {
}
virtual ~GetRuntimeStatsRequest() throw();
+ int32_t maxWeight;
+ int32_t maxCreateTime;
- bool operator == (const GetRuntimeStatsRequest & /* rhs */) const
+ void __set_maxWeight(const int32_t val);
+
+ void __set_maxCreateTime(const int32_t val);
+
+ bool operator == (const GetRuntimeStatsRequest & rhs) const
{
+ if (!(maxWeight == rhs.maxWeight))
+ return false;
+ if (!(maxCreateTime == rhs.maxCreateTime))
+ return false;
return true;
}
bool operator != (const GetRuntimeStatsRequest &rhs) const {
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java
index e5ce97e..593b7bf 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java
@@ -38,6 +38,8 @@ import org.slf4j.LoggerFactory;
@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class GetRuntimeStatsRequest implements org.apache.thrift.TBase<GetRuntimeStatsRequest, GetRuntimeStatsRequest._Fields>, java.io.Serializable, Cloneable, Comparable<GetRuntimeStatsRequest> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetRuntimeStatsRequest");
+ private static final org.apache.thrift.protocol.TField MAX_WEIGHT_FIELD_DESC = new org.apache.thrift.protocol.TField("maxWeight", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField MAX_CREATE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("maxCreateTime", org.apache.thrift.protocol.TType.I32, (short)2);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -45,10 +47,13 @@ import org.slf4j.LoggerFactory;
schemes.put(TupleScheme.class, new GetRuntimeStatsRequestTupleSchemeFactory());
}
+ private int maxWeight; // required
+ private int maxCreateTime; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
+ MAX_WEIGHT((short)1, "maxWeight"),
+ MAX_CREATE_TIME((short)2, "maxCreateTime");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -63,6 +68,10 @@ import org.slf4j.LoggerFactory;
*/
public static _Fields findByThriftId(int fieldId) {
switch(fieldId) {
+ case 1: // MAX_WEIGHT
+ return MAX_WEIGHT;
+ case 2: // MAX_CREATE_TIME
+ return MAX_CREATE_TIME;
default:
return null;
}
@@ -101,9 +110,18 @@ import org.slf4j.LoggerFactory;
return _fieldName;
}
}
+
+ // isset id assignments
+ private static final int __MAXWEIGHT_ISSET_ID = 0;
+ private static final int __MAXCREATETIME_ISSET_ID = 1;
+ private byte __isset_bitfield = 0;
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.MAX_WEIGHT, new org.apache.thrift.meta_data.FieldMetaData("maxWeight", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.MAX_CREATE_TIME, new org.apache.thrift.meta_data.FieldMetaData("maxCreateTime", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetRuntimeStatsRequest.class, metaDataMap);
}
@@ -111,10 +129,24 @@ import org.slf4j.LoggerFactory;
public GetRuntimeStatsRequest() {
}
+ public GetRuntimeStatsRequest(
+ int maxWeight,
+ int maxCreateTime)
+ {
+ this();
+ this.maxWeight = maxWeight;
+ setMaxWeightIsSet(true);
+ this.maxCreateTime = maxCreateTime;
+ setMaxCreateTimeIsSet(true);
+ }
+
/**
* Performs a deep copy on <i>other</i>.
*/
public GetRuntimeStatsRequest(GetRuntimeStatsRequest other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.maxWeight = other.maxWeight;
+ this.maxCreateTime = other.maxCreateTime;
}
public GetRuntimeStatsRequest deepCopy() {
@@ -123,15 +155,85 @@ import org.slf4j.LoggerFactory;
@Override
public void clear() {
+ setMaxWeightIsSet(false);
+ this.maxWeight = 0;
+ setMaxCreateTimeIsSet(false);
+ this.maxCreateTime = 0;
+ }
+
+ public int getMaxWeight() {
+ return this.maxWeight;
+ }
+
+ public void setMaxWeight(int maxWeight) {
+ this.maxWeight = maxWeight;
+ setMaxWeightIsSet(true);
+ }
+
+ public void unsetMaxWeight() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID);
+ }
+
+ /** Returns true if field maxWeight is set (has been assigned a value) and false otherwise */
+ public boolean isSetMaxWeight() {
+ return EncodingUtils.testBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID);
+ }
+
+ public void setMaxWeightIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID, value);
+ }
+
+ public int getMaxCreateTime() {
+ return this.maxCreateTime;
+ }
+
+ public void setMaxCreateTime(int maxCreateTime) {
+ this.maxCreateTime = maxCreateTime;
+ setMaxCreateTimeIsSet(true);
+ }
+
+ public void unsetMaxCreateTime() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID);
+ }
+
+ /** Returns true if field maxCreateTime is set (has been assigned a value) and false otherwise */
+ public boolean isSetMaxCreateTime() {
+ return EncodingUtils.testBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID);
+ }
+
+ public void setMaxCreateTimeIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID, value);
}
public void setFieldValue(_Fields field, Object value) {
switch (field) {
+ case MAX_WEIGHT:
+ if (value == null) {
+ unsetMaxWeight();
+ } else {
+ setMaxWeight((Integer)value);
+ }
+ break;
+
+ case MAX_CREATE_TIME:
+ if (value == null) {
+ unsetMaxCreateTime();
+ } else {
+ setMaxCreateTime((Integer)value);
+ }
+ break;
+
}
}
public Object getFieldValue(_Fields field) {
switch (field) {
+ case MAX_WEIGHT:
+ return getMaxWeight();
+
+ case MAX_CREATE_TIME:
+ return getMaxCreateTime();
+
}
throw new IllegalStateException();
}
@@ -143,6 +245,10 @@ import org.slf4j.LoggerFactory;
}
switch (field) {
+ case MAX_WEIGHT:
+ return isSetMaxWeight();
+ case MAX_CREATE_TIME:
+ return isSetMaxCreateTime();
}
throw new IllegalStateException();
}
@@ -160,6 +266,24 @@ import org.slf4j.LoggerFactory;
if (that == null)
return false;
+ boolean this_present_maxWeight = true;
+ boolean that_present_maxWeight = true;
+ if (this_present_maxWeight || that_present_maxWeight) {
+ if (!(this_present_maxWeight && that_present_maxWeight))
+ return false;
+ if (this.maxWeight != that.maxWeight)
+ return false;
+ }
+
+ boolean this_present_maxCreateTime = true;
+ boolean that_present_maxCreateTime = true;
+ if (this_present_maxCreateTime || that_present_maxCreateTime) {
+ if (!(this_present_maxCreateTime && that_present_maxCreateTime))
+ return false;
+ if (this.maxCreateTime != that.maxCreateTime)
+ return false;
+ }
+
return true;
}
@@ -167,6 +291,16 @@ import org.slf4j.LoggerFactory;
public int hashCode() {
List<Object> list = new ArrayList<Object>();
+ boolean present_maxWeight = true;
+ list.add(present_maxWeight);
+ if (present_maxWeight)
+ list.add(maxWeight);
+
+ boolean present_maxCreateTime = true;
+ list.add(present_maxCreateTime);
+ if (present_maxCreateTime)
+ list.add(maxCreateTime);
+
return list.hashCode();
}
@@ -178,6 +312,26 @@ import org.slf4j.LoggerFactory;
int lastComparison = 0;
+ lastComparison = Boolean.valueOf(isSetMaxWeight()).compareTo(other.isSetMaxWeight());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMaxWeight()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxWeight, other.maxWeight);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetMaxCreateTime()).compareTo(other.isSetMaxCreateTime());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMaxCreateTime()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxCreateTime, other.maxCreateTime);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -198,12 +352,27 @@ import org.slf4j.LoggerFactory;
StringBuilder sb = new StringBuilder("GetRuntimeStatsRequest(");
boolean first = true;
+ sb.append("maxWeight:");
+ sb.append(this.maxWeight);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("maxCreateTime:");
+ sb.append(this.maxCreateTime);
+ first = false;
sb.append(")");
return sb.toString();
}
public void validate() throws org.apache.thrift.TException {
// check for required fields
+ if (!isSetMaxWeight()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxWeight' is unset! Struct:" + toString());
+ }
+
+ if (!isSetMaxCreateTime()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxCreateTime' is unset! Struct:" + toString());
+ }
+
// check for sub-struct validity
}
@@ -217,6 +386,8 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -241,6 +412,22 @@ import org.slf4j.LoggerFactory;
break;
}
switch (schemeField.id) {
+ case 1: // MAX_WEIGHT
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.maxWeight = iprot.readI32();
+ struct.setMaxWeightIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // MAX_CREATE_TIME
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.maxCreateTime = iprot.readI32();
+ struct.setMaxCreateTimeIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -254,6 +441,12 @@ import org.slf4j.LoggerFactory;
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(MAX_WEIGHT_FIELD_DESC);
+ oprot.writeI32(struct.maxWeight);
+ oprot.writeFieldEnd();
+ oprot.writeFieldBegin(MAX_CREATE_TIME_FIELD_DESC);
+ oprot.writeI32(struct.maxCreateTime);
+ oprot.writeFieldEnd();
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -271,11 +464,17 @@ import org.slf4j.LoggerFactory;
@Override
public void write(org.apache.thrift.protocol.TProtocol prot, GetRuntimeStatsRequest struct) throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeI32(struct.maxWeight);
+ oprot.writeI32(struct.maxCreateTime);
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, GetRuntimeStatsRequest struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.maxWeight = iprot.readI32();
+ struct.setMaxWeightIsSet(true);
+ struct.maxCreateTime = iprot.readI32();
+ struct.setMaxCreateTimeIsSet(true);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
index c9d1051..1abedc9 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
@@ -6,11 +6,10 @@
*/
package org.apache.hadoop.hive.metastore.api;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Field;
import org.apache.thrift.scheme.IScheme;
import org.apache.thrift.scheme.SchemeFactory;
import org.apache.thrift.scheme.StandardScheme;
+
import org.apache.thrift.scheme.TupleScheme;
import org.apache.thrift.protocol.TTupleProtocol;
import org.apache.thrift.protocol.TProtocolException;
@@ -35,9 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@org.apache.hadoop.classification.InterfaceAudience.Public
-@org.apache.hadoop.classification.InterfaceStability.Stable
-public class hive_metastoreConstants {
+@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class hive_metastoreConstants {
public static final String DDL_TIME = "transient_lastDdlTime";
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index c9ebfaf..9f6cc0e 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -29846,12 +29846,36 @@ class RuntimeStat {
class GetRuntimeStatsRequest {
static $_TSPEC;
+ /**
+ * @var int
+ */
+ public $maxWeight = null;
+ /**
+ * @var int
+ */
+ public $maxCreateTime = null;
- public function __construct() {
+ public function __construct($vals=null) {
if (!isset(self::$_TSPEC)) {
self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'maxWeight',
+ 'type' => TType::I32,
+ ),
+ 2 => array(
+ 'var' => 'maxCreateTime',
+ 'type' => TType::I32,
+ ),
);
}
+ if (is_array($vals)) {
+ if (isset($vals['maxWeight'])) {
+ $this->maxWeight = $vals['maxWeight'];
+ }
+ if (isset($vals['maxCreateTime'])) {
+ $this->maxCreateTime = $vals['maxCreateTime'];
+ }
+ }
}
public function getName() {
@@ -29873,6 +29897,20 @@ class GetRuntimeStatsRequest {
}
switch ($fid)
{
+ case 1:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->maxWeight);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->maxCreateTime);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -29886,6 +29924,16 @@ class GetRuntimeStatsRequest {
public function write($output) {
$xfer = 0;
$xfer += $output->writeStructBegin('GetRuntimeStatsRequest');
+ if ($this->maxWeight !== null) {
+ $xfer += $output->writeFieldBegin('maxWeight', TType::I32, 1);
+ $xfer += $output->writeI32($this->maxWeight);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->maxCreateTime !== null) {
+ $xfer += $output->writeFieldBegin('maxCreateTime', TType::I32, 2);
+ $xfer += $output->writeI32($this->maxCreateTime);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index b1e577a..69579e2 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -21332,10 +21332,22 @@ class RuntimeStat:
return not (self == other)
class GetRuntimeStatsRequest:
+ """
+ Attributes:
+ - maxWeight
+ - maxCreateTime
+ """
thrift_spec = (
+ None, # 0
+ (1, TType.I32, 'maxWeight', None, None, ), # 1
+ (2, TType.I32, 'maxCreateTime', None, None, ), # 2
)
+ def __init__(self, maxWeight=None, maxCreateTime=None,):
+ self.maxWeight = maxWeight
+ self.maxCreateTime = maxCreateTime
+
def read(self, iprot):
if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
@@ -21345,6 +21357,16 @@ class GetRuntimeStatsRequest:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
+ if fid == 1:
+ if ftype == TType.I32:
+ self.maxWeight = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.maxCreateTime = iprot.readI32()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -21355,15 +21377,29 @@ class GetRuntimeStatsRequest:
oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
return
oprot.writeStructBegin('GetRuntimeStatsRequest')
+ if self.maxWeight is not None:
+ oprot.writeFieldBegin('maxWeight', TType.I32, 1)
+ oprot.writeI32(self.maxWeight)
+ oprot.writeFieldEnd()
+ if self.maxCreateTime is not None:
+ oprot.writeFieldBegin('maxCreateTime', TType.I32, 2)
+ oprot.writeI32(self.maxCreateTime)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
+ if self.maxWeight is None:
+ raise TProtocol.TProtocolException(message='Required field maxWeight is unset!')
+ if self.maxCreateTime is None:
+ raise TProtocol.TProtocolException(message='Required field maxCreateTime is unset!')
return
def __hash__(self):
value = 17
+ value = (value * 31) ^ hash(self.maxWeight)
+ value = (value * 31) ^ hash(self.maxCreateTime)
return value
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 2687ce5..d7ebaaf 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -4849,14 +4849,19 @@ end
class GetRuntimeStatsRequest
include ::Thrift::Struct, ::Thrift::Struct_Union
+ MAXWEIGHT = 1
+ MAXCREATETIME = 2
FIELDS = {
-
+ MAXWEIGHT => {:type => ::Thrift::Types::I32, :name => 'maxWeight'},
+ MAXCREATETIME => {:type => ::Thrift::Types::I32, :name => 'maxCreateTime'}
}
def struct_fields; FIELDS; end
def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxWeight is unset!') unless @maxWeight
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxCreateTime is unset!') unless @maxCreateTime
end
::Thrift::Struct.generate_accessors self
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 397a081..b0d5d52 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -8461,7 +8461,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("get_runtime_stats");
Exception ex = null;
try {
- List<RuntimeStat> res = getMS().getRuntimeStats();
+ List<RuntimeStat> res = getMS().getRuntimeStats(rqst.getMaxWeight(), rqst.getMaxCreateTime());
return res;
} catch (MetaException e) {
LOG.error("Caught exception", e);
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 1138ed3..e60c5e1 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -3249,7 +3248,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws TException {
- return client.get_runtime_stats(new GetRuntimeStatsRequest());
+ public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException {
+ GetRuntimeStatsRequest req = new GetRuntimeStatsRequest();
+ req.setMaxWeight(maxWeight);
+ req.setMaxCreateTime(maxCreateTime);
+ return client.get_runtime_stats(req);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 72b814d..ebdf58f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.classification.RetrySemantics;
import org.apache.hadoop.hive.metastore.annotation.NoReconnect;
@@ -3648,6 +3647,6 @@ public interface IMetaStoreClient {
void addRuntimeStat(RuntimeStat stat) throws TException;
/** Reads runtime statistics. */
- List<RuntimeStat> getRuntimeStats() throws TException;
+ List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 2bd7c21..9c94d7e 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -55,8 +55,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
import javax.jdo.JDOCanRetryException;
import javax.jdo.JDODataStoreException;
import javax.jdo.JDOException;
@@ -11616,41 +11614,47 @@ public class ObjectStore implements RawStore, Configurable {
public void addRuntimeStat(RuntimeStat stat) throws MetaException {
LOG.debug("runtimeStat: " + stat);
MRuntimeStat mStat = MRuntimeStat.fromThrift(stat);
- pm.makePersistent(mStat);
+ boolean committed = false;
+ openTransaction();
+ try {
+ pm.makePersistent(mStat);
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
}
@Override
- public int deleteRuntimeStats(int maxRetainedWeight, int maxRetainSecs) throws MetaException {
- List<MRuntimeStat> all = getMRuntimeStats();
- int retentionTime = 0;
- if (maxRetainSecs >= 0) {
- retentionTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs;
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
+ if (maxRetainSecs < 0) {
+ LOG.warn("runtime stats retention is disabled");
+ return 0;
}
- if (maxRetainedWeight < 0) {
- maxRetainedWeight = Integer.MAX_VALUE;
- }
-
- Object maxIdToRemove = null;
- long totalWeight = 0;
- int deleted = 0;
- for (MRuntimeStat mRuntimeStat : all) {
- totalWeight += mRuntimeStat.getWeight();
- if (totalWeight > maxRetainedWeight || mRuntimeStat.getCreatedTime() < retentionTime) {
- LOG.debug("removing runtime stat: " + mRuntimeStat);
- pm.deletePersistent(mRuntimeStat);
- deleted++;
+ boolean committed = false;
+ try {
+ openTransaction();
+ int maxCreateTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs;
+ Query q = pm.newQuery(MRuntimeStat.class);
+ q.setFilter("createTime <= maxCreateTime");
+ q.declareParameters("int maxCreateTime");
+ long deleted = q.deletePersistentAll(maxCreateTime);
+ committed = commitTransaction();
+ return (int) deleted;
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
}
}
- return deleted;
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws MetaException {
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
boolean committed = false;
try {
openTransaction();
- List<MRuntimeStat> mStats = getMRuntimeStats();
- List<RuntimeStat> stats = mStats.stream().map(MRuntimeStat::toThrift).collect(Collectors.toList());
+ List<RuntimeStat> stats = getMRuntimeStats(maxEntries, maxCreateTime);
committed = commitTransaction();
return stats;
} finally {
@@ -11660,12 +11664,27 @@ public class ObjectStore implements RawStore, Configurable {
}
}
- private List<MRuntimeStat> getMRuntimeStats() {
+ private List<RuntimeStat> getMRuntimeStats(int maxEntries, int maxCreateTime) {
Query<MRuntimeStat> query = pm.newQuery(MRuntimeStat.class);
query.setOrdering("createTime descending");
+ if (maxCreateTime > 0) {
+ query.setFilter("createTime < "+maxCreateTime);
+ }
+ if (maxEntries < 0) {
+ maxEntries = Integer.MAX_VALUE;
+ }
+ List<RuntimeStat> ret = new ArrayList<>();
List<MRuntimeStat> res = (List<MRuntimeStat>) query.execute();
- pm.retrieveAll(res);
- return res;
+ int totalEntries = 0;
+ for (MRuntimeStat mRuntimeStat : res) {
+ pm.retrieve(mRuntimeStat);
+ totalEntries += mRuntimeStat.getWeight();
+ ret.add(MRuntimeStat.toThrift(mRuntimeStat));
+ if (totalEntries >= maxEntries) {
+ break;
+ }
+ }
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index 2c9f2e5..3719c39 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -1630,8 +1630,9 @@ public interface RawStore extends Configurable {
void addRuntimeStat(RuntimeStat stat) throws MetaException;
/** Reads runtime statistic entries. */
- List<RuntimeStat> getRuntimeStats() throws MetaException;
+ List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException;
/** Removes outdated statistics. */
- int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException;
+ int deleteRuntimeStats(int maxRetainSecs) throws MetaException;
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java
index 202058e..ccec93d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java
@@ -53,9 +53,8 @@ public class RuntimeStatsCleanerTask implements MetastoreTaskThread {
try {
RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf);
- int maxRetained = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_ENTRIES);
int maxRetainSecs=(int) MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_AGE, TimeUnit.SECONDS);
- int deleteCnt = ms.deleteRuntimeStats(maxRetained, maxRetainSecs);
+ int deleteCnt = ms.deleteRuntimeStats(maxRetainSecs);
if (deleteCnt > 0L){
LOG.info("Number of deleted entries: " + deleteCnt);
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 92d000b..30d4bec 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -2477,12 +2477,12 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws MetaException {
- return rawStore.getRuntimeStats();
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
+ return rawStore.getRuntimeStats(maxEntries, maxCreateTime);
}
@Override
- public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException {
- return rawStore.deleteRuntimeStats(maxRetained, maxRetainSecs);
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
+ return rawStore.deleteRuntimeStats(maxRetainSecs);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 35aa40c..114d5da 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -583,9 +583,6 @@ public class MetastoreConf {
TimeUnit.SECONDS, "Frequency at which timer task runs to remove outdated runtime stat entries."),
RUNTIME_STATS_MAX_AGE("runtime.stats.max.age", "hive.metastore.runtime.stats.max.age", 86400 * 3, TimeUnit.SECONDS,
"Stat entries which are older than this are removed."),
- RUNTIME_STATS_MAX_ENTRIES("runtime.stats.max.entries", "hive.metastore.runtime.stats.max.entries", 100_000,
- "Maximum number of runtime stats to keep; unit is operator stat infos - a complicated query has ~100 of these."
- + "See also: hive.query.reexecution.stats.cache.size"),
// Parameters for exporting metadata on table drop (requires the use of the)
// org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index c56a4f9..ccc3c93 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -1524,6 +1524,8 @@ struct RuntimeStat {
}
struct GetRuntimeStatsRequest {
+ 1: required i32 maxWeight,
+ 2: required i32 maxCreateTime
}
// Exceptions.
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index defc68f..3895c3d 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -1171,12 +1171,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws MetaException {
- return objectStore.getRuntimeStats();
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
+ return objectStore.getRuntimeStats(maxEntries, maxCreateTime);
}
@Override
- public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException {
- return objectStore.deleteRuntimeStats(maxRetained, maxRetainSecs);
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
+ return objectStore.deleteRuntimeStats(maxRetainSecs);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 20c5d8a..cdfa001 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -1157,12 +1157,12 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws MetaException {
+ public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException {
return Collections.emptyList();
}
@Override
- public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException {
+ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException {
return 0;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 8ae899f..4960e3b 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -3359,7 +3358,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
- public List<RuntimeStat> getRuntimeStats() throws TException {
+ public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java
index 2db7a8b..0b91ed4 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore.client;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.ObjectStore;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.api.RuntimeStat;
import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService;
@@ -40,8 +41,10 @@ import static org.junit.Assert.assertNotNull;
public class TestRuntimeStats extends MetaStoreClientTest {
private final AbstractMetaStoreService metaStore;
private IMetaStoreClient client;
+ private String metastoreName;
public TestRuntimeStats(String name, AbstractMetaStoreService metaStore) throws Exception {
+ this.metastoreName = name;
this.metaStore = metaStore;
this.metaStore.start();
}
@@ -60,14 +63,14 @@ public class TestRuntimeStats extends MetaStoreClientTest {
@Test
public void testRuntimeStatHandling() throws Exception {
- List<RuntimeStat> rs0 = client.getRuntimeStats();
+ List<RuntimeStat> rs0 = getRuntimeStats();
assertNotNull(rs0);
assertEquals(0, rs0.size());
RuntimeStat stat = createStat(1);
client.addRuntimeStat(stat);
- List<RuntimeStat> rs1 = client.getRuntimeStats();
+ List<RuntimeStat> rs1 = getRuntimeStats();
assertNotNull(rs1);
assertEquals(1, rs1.size());
assertArrayEquals(stat.getPayload(), rs1.get(0).getPayload());
@@ -79,11 +82,59 @@ public class TestRuntimeStats extends MetaStoreClientTest {
client.addRuntimeStat(createStat(3));
client.addRuntimeStat(createStat(4));
- List<RuntimeStat> rs2 = client.getRuntimeStats();
+ List<RuntimeStat> rs2 = getRuntimeStats();
assertEquals(4, rs2.size());
}
+ @Test
+ public void testCleanup() throws Exception {
+ ObjectStore objStore = new ObjectStore();
+ objStore.setConf(metaStore.getConf());
+ objStore.deleteRuntimeStats(0);
+ objStore.addRuntimeStat(createStat(1));
+ Thread.sleep(2000);
+ objStore.addRuntimeStat(createStat(2));
+ int deleted = objStore.deleteRuntimeStats(1);
+ assertEquals(1, deleted);
+
+ List<RuntimeStat> all = getRuntimeStats();
+ assertEquals(1, all.size());
+ assertEquals(2, all.get(0).getWeight());
+
+ }
+
+ @Test
+ public void testReading() throws Exception {
+ ObjectStore objStore = new ObjectStore();
+ objStore.setConf(metaStore.getConf());
+ objStore.deleteRuntimeStats(0);
+ objStore.addRuntimeStat(createStat(1));
+ Thread.sleep(1000);
+ objStore.addRuntimeStat(createStat(2));
+ Thread.sleep(1000);
+ objStore.addRuntimeStat(createStat(3));
+
+ List<RuntimeStat> g0 = client.getRuntimeStats(3, -1);
+ assertEquals(1, g0.size());
+ assertEquals(3, g0.get(0).getWeight());
+ int ct = g0.get(0).getCreateTime();
+ List<RuntimeStat> g1 = client.getRuntimeStats(3, ct);
+
+ assertEquals(2, g1.size());
+ assertEquals(2, g1.get(0).getWeight());
+ assertEquals(1, g1.get(1).getWeight());
+ int ct1 = g1.get(1).getCreateTime();
+ List<RuntimeStat> g2 = client.getRuntimeStats(3, ct1);
+
+ assertEquals(0, g2.size());
+
+ }
+
+ private List<RuntimeStat> getRuntimeStats() throws Exception {
+ return client.getRuntimeStats(-1, -1);
+ }
+
private RuntimeStat createStat(int w) {
byte[] payload = new byte[w];
@@ -97,4 +148,5 @@ public class TestRuntimeStats extends MetaStoreClientTest {
return stat;
}
+
}