You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by an...@apache.org on 2020/06/23 04:39:23 UTC
[hive] branch master updated: HIVE-23668: Clean up Task for Hive
Metrics (Aasha Medhi, reviewed by Pravin Kumar Sinha)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7454393 HIVE-23668: Clean up Task for Hive Metrics (Aasha Medhi, reviewed by Pravin Kumar Sinha)
7454393 is described below
commit 74543935c1be9f9dbd619dfac69eefaa7ba19986
Author: Anishek Agarwal <an...@gmail.com>
AuthorDate: Tue Jun 23 10:09:11 2020 +0530
HIVE-23668: Clean up Task for Hive Metrics (Aasha Medhi, reviewed by Pravin Kumar Sinha)
---
.../hcatalog/listener/DummyRawStoreFailEvent.java | 5 ++
.../parse/TestScheduledReplicationScenarios.java | 2 +-
.../hive/ql/parse/repl/metric/MetricSink.java | 2 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 12 +++-
.../src/main/thrift/hive_metastore.thrift | 2 +-
.../apache/hadoop/hive/metastore/ObjectStore.java | 23 +++++++
.../org/apache/hadoop/hive/metastore/RawStore.java | 2 +
.../metastore/ReplicationMetricsMaintTask.java | 77 ++++++++++++++++++++++
.../hadoop/hive/metastore/cache/CachedStore.java | 5 ++
.../hive/metastore/model/MReplicationMetrics.java | 9 +++
.../src/main/resources/package.jdo | 3 +
.../src/main/sql/derby/hive-schema-4.0.0.derby.sql | 1 +
.../sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql | 1 +
.../src/main/sql/mssql/hive-schema-4.0.0.mssql.sql | 3 +-
.../sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql | 3 +-
.../src/main/sql/mysql/hive-schema-4.0.0.mysql.sql | 1 +
.../sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql | 1 +
.../main/sql/oracle/hive-schema-4.0.0.oracle.sql | 3 +-
.../sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql | 3 +-
.../sql/postgres/hive-schema-4.0.0.postgres.sql | 1 +
.../postgres/upgrade-3.2.0-to-4.0.0.postgres.sql | 1 +
.../metastore/DummyRawStoreControlledCommit.java | 5 ++
.../metastore/DummyRawStoreForJdoConnection.java | 5 ++
.../metastore/client/TestReplicationMetrics.java | 45 ++++++++++++-
24 files changed, 203 insertions(+), 12 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 34fb318..a4b2f52 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -1390,6 +1390,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
+ public int deleteReplicationMetrics(int maxRetainSecs) {
+ return objectStore.deleteReplicationMetrics(maxRetainSecs);
+ }
+
+ @Override
public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException {
throw new RuntimeException("unimplemented");
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
index 017607e..fd0a214 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
@@ -58,7 +58,7 @@ import java.util.ArrayList;
* TestScheduledReplicationScenarios - test scheduled replication .
*/
public class TestScheduledReplicationScenarios extends BaseReplicationScenariosAcidTables {
- private static final long DEFAULT_PROBE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes
+ private static final long DEFAULT_PROBE_TIMEOUT = 5 * 60 * 1000L; // 5 minutes
@BeforeClass
public static void classLevelSetup() throws Exception {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java
index 88a7b07..a856c76 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java
@@ -71,7 +71,7 @@ public final class MetricSink {
long getFrequencyInSecs() {
//Metastore conf is in minutes
- return MetastoreConf.getLongVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY) * 60;
+ return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY, TimeUnit.MINUTES) * 60;
}
public synchronized void tearDown() {
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 02fb9cf..ca9cd7d 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -981,9 +981,16 @@ public class MetastoreConf {
+ " and the frequency of persisting the metrics to persistent storage. "
),
REPL_METRICS_UPDATE_FREQUENCY("metastore.repl.metrics.update.frequency",
- "hive.repl.metrics.update.frequency", 1 /*1 minute */,
+ "hive.repl.metrics.update.frequency", 1L, TimeUnit.MINUTES /*1 minute */,
"Frequency at which replication Metrics will be stored in persistent storage. "
),
+ REPL_METRICS_CLEANUP_FREQUENCY("metastore.repl.metrics.cleanup.frequency",
+ "hive.metastore.repl.metrics.cleanup.frequency", 1, TimeUnit.DAYS,
+ "Interval of scheduled metrics clean up task which removes metrics above max age; Max age is"
+ + " defined by the config metastore.repl.metrics.max.age. The max age should be greater than this frequency"),
+ REPL_METRICS_MAX_AGE("metastore.repl.metrics.max.age",
+ "hive.metastore.repl.metrics.max.age", 7, TimeUnit.DAYS,
+ "Maximal age of a replication metrics entry before it is removed."),
SCHEMA_INFO_CLASS("metastore.schema.info.class", "hive.metastore.schema.info.class",
"org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo",
"Fully qualified class name for the metastore schema information class \n"
@@ -1077,7 +1084,8 @@ public class MetastoreConf {
EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," +
"org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," +
"org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask" + ","
- + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask",
+ + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask" + ","
+ + "org.apache.hadoop.hive.metastore.ReplicationMetricsMaintTask",
"Comma separated list of tasks that will be started in separate threads. These will " +
"always be started, regardless of whether the metastore is running in embedded mode " +
"or in server mode. They must implement " + METASTORE_TASK_THREAD_CLASS),
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index 903b531..d6bfb89 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -2039,7 +2039,7 @@ struct ReplicationMetrics{
2: required string policy,
3: required i64 dumpExecutionId,
4: optional string metadata,
- 5: optional string progress,
+ 5: optional string progress
}
struct ReplicationMetricList{
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 5af7169..c472e93 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -13104,6 +13104,7 @@ public class ObjectStore implements RawStore, Configurable {
mReplicationMetrics.setDumpExecutionId(replicationMetric.getDumpExecutionId());
mReplicationMetrics.setScheduledExecutionId(replicationMetric.getScheduledExecutionId());
mReplicationMetrics.setPolicy(replicationMetric.getPolicy());
+ mReplicationMetrics.setStartTime((int) (System.currentTimeMillis()/1000));
}
if (!StringUtils.isEmpty(replicationMetric.getMetadata())) {
mReplicationMetrics.setMetadata(replicationMetric.getMetadata());
@@ -13142,6 +13143,28 @@ public class ObjectStore implements RawStore, Configurable {
}
}
+ @Override
+ public int deleteReplicationMetrics(int maxRetainSecs) {
+ if (maxRetainSecs < 0) {
+ LOG.debug("replication metrics deletion is disabled");
+ return 0;
+ }
+ boolean committed = false;
+ Query q = null;
+ try {
+ openTransaction();
+ int maxCreateTime = (int) ((System.currentTimeMillis() / 1000) - maxRetainSecs);
+ q = pm.newQuery(MReplicationMetrics.class);
+ q.setFilter("startTime <= maxCreateTime");
+ q.declareParameters("long maxCreateTime");
+ long deleted = q.deletePersistentAll(maxCreateTime);
+ committed = commitTransaction();
+ return (int) deleted;
+ } finally {
+ rollbackAndCleanup(committed, q);
+ }
+ }
+
private ReplicationMetricList getMReplicationMetrics(String policy) {
ReplicationMetricList ret = new ReplicationMetricList();
if (StringUtils.isEmpty(policy)) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index 6534750..d456255 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -1857,6 +1857,8 @@ public interface RawStore extends Configurable {
*/
ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest replicationMetricsRequest);
+ int deleteReplicationMetrics(int maxRetainSecs);
+
int deleteScheduledExecutions(int maxRetainSecs);
int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java
new file mode 100644
index 0000000..4ba968f
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Metastore task to remove old replication metrics.
+ */
+public class ReplicationMetricsMaintTask implements MetastoreTaskThread {
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationMetricsMaintTask.class);
+
+ private Configuration conf;
+
+ @Override
+ public long initialDelay(TimeUnit unit) {
+ // no delay before the first execution;
+ // after an ungracefull shutdown it might take time to notice that in-flight scheduled queries are not running anymore
+ return 0;
+ }
+
+ @Override
+ public long runFrequency(TimeUnit unit) {
+ return MetastoreConf.getTimeVar(conf, ConfVars.REPL_METRICS_CLEANUP_FREQUENCY,
+ TimeUnit.DAYS);
+ }
+
+ @Override
+ public void setConf(Configuration configuration) {
+ conf = configuration;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!MetastoreConf.getBoolVar(conf, ConfVars.SCHEDULED_QUERIES_ENABLED)) {
+ return;
+ }
+ RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf);
+ int maxRetainSecs = (int) TimeUnit.DAYS.toSeconds(MetastoreConf.getTimeVar(conf,
+ ConfVars.REPL_METRICS_MAX_AGE, TimeUnit.DAYS));
+ int deleteCnt = ms.deleteScheduledExecutions(maxRetainSecs);
+ if (deleteCnt > 0L){
+ LOG.info("Number of deleted entries: " + deleteCnt);
+ }
+ } catch (Exception e) {
+ LOG.error("Exception while trying to delete: " + e.getMessage(), e);
+ }
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index c8e230b..16df3c3 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -2900,6 +2900,11 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
+ public int deleteReplicationMetrics(int maxRetainSecs) {
+ return rawStore.deleteReplicationMetrics(maxRetainSecs);
+ }
+
+ @Override
public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException {
return rawStore.getScheduledQuery(scheduleKey);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java
index 463be25..5fe3129 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java
@@ -29,6 +29,7 @@ public class MReplicationMetrics {
private long dumpExecutionId;
private String metadata;
private String progress;
+ private int startTime;
public MReplicationMetrics() {
}
@@ -82,4 +83,12 @@ public class MReplicationMetrics {
public void setProgress(String progress) {
this.progress = progress;
}
+
+ public int getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(int startTime) {
+ this.startTime = startTime;
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/resources/package.jdo b/standalone-metastore/metastore-server/src/main/resources/package.jdo
index c88ed32..0252dd0 100644
--- a/standalone-metastore/metastore-server/src/main/resources/package.jdo
+++ b/standalone-metastore/metastore-server/src/main/resources/package.jdo
@@ -1539,6 +1539,9 @@
<field name="progress">
<column name="RM_PROGRESS" jdbc-type="varchar" length="4000" allows-null="true"/>
</field>
+ <field name="startTime">
+ <column name="RM_START_TIME" jdbc-type="integer" allows-null="false"/>
+ </field>
<index name="PolicyIndex">
<column name="RM_POLICY"/>
</index>
diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql
index 811f2b1..c64805d 100644
--- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql
@@ -779,6 +779,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" (
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(4000),
"RM_PROGRESS" varchar(4000),
+ "RM_START_TIME" integer not null,
PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID")
);
diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql
index aeea23f..a9c2cab 100644
--- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql
@@ -94,6 +94,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" (
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(4000),
"RM_PROGRESS" varchar(4000),
+ "RM_START_TIME" integer not null,
PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID")
);
diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
index cba72cd..a59b646 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
@@ -1326,7 +1326,8 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_POLICY" varchar(256) NOT NULL,
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(max),
- "RM_PROGRESS" varchar(max)
+ "RM_PROGRESS" varchar(max),
+ "RM_START_TIME" integer NOT NULL
);
-- Create indexes for the replication metrics table
diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql
index fa1250c..0826c1e 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql
@@ -129,7 +129,8 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_POLICY" varchar(256) NOT NULL,
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(max),
- "RM_PROGRESS" varchar(max)
+ "RM_PROGRESS" varchar(max),
+ "RM_START_TIME" integer NOT NULL
);
-- Create indexes for the replication metrics table
diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
index e036b31..0d0374c 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
@@ -1251,6 +1251,7 @@ CREATE TABLE REPLICATION_METRICS (
RM_DUMP_EXECUTION_ID bigint NOT NULL,
RM_METADATA varchar(4000),
RM_PROGRESS varchar(4000),
+ RM_START_TIME integer NOT NULL,
PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID)
);
diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql
index c1c888b..acf09c1 100644
--- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql
@@ -101,6 +101,7 @@ CREATE TABLE REPLICATION_METRICS (
RM_DUMP_EXECUTION_ID bigint NOT NULL,
RM_METADATA varchar(4000),
RM_PROGRESS varchar(4000),
+ RM_START_TIME integer NOT NULL,
PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID)
);
diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
index c7bf0ba..bc757bf 100644
--- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
@@ -1224,7 +1224,8 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_POLICY" varchar2(256) NOT NULL,
"RM_DUMP_EXECUTION_ID" number NOT NULL,
"RM_METADATA" varchar2(4000),
- "RM_PROGRESS" varchar2(4000)
+ "RM_PROGRESS" varchar2(4000),
+ "RM_START_TIME" integer NOT NULL
);
--Create indexes for the replication metrics table
diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql
index 6146710..84db436 100644
--- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql
@@ -98,7 +98,8 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_POLICY" varchar2(256) NOT NULL,
"RM_DUMP_EXECUTION_ID" number NOT NULL,
"RM_METADATA" varchar2(4000),
- "RM_PROGRESS" varchar2(4000)
+ "RM_PROGRESS" varchar2(4000),
+ "RM_START_TIME" integer NOT NULL
);
--Create indexes for the replication metrics table
diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
index bdbe88f..9b03d73 100644
--- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
@@ -1913,6 +1913,7 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(4000),
"RM_PROGRESS" varchar(4000),
+ "RM_START_TIME" integer NOT NULL,
PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID")
);
diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql
index f6caac8..53f671c 100644
--- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql
+++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql
@@ -225,6 +225,7 @@ CREATE TABLE "REPLICATION_METRICS" (
"RM_DUMP_EXECUTION_ID" bigint NOT NULL,
"RM_METADATA" varchar(4000),
"RM_PROGRESS" varchar(4000),
+ "RM_START_TIME" integer NOT NULL,
PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID")
);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index d7b2d65..a779813 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -1339,6 +1339,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public int deleteReplicationMetrics(int maxRetainSecs) {
+ return objectStore.deleteReplicationMetrics(maxRetainSecs);
+ }
+
+ @Override
public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws NoSuchObjectException {
return objectStore.getScheduledQuery(scheduleKey);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 1958607..1cbc86f 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -1322,6 +1322,11 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public int deleteReplicationMetrics(int maxRetainSecs) {
+ throw new RuntimeException("unimplemented");
+ }
+
+ @Override
public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) {
throw new RuntimeException("unimplemented");
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java
index f8c8e1f..d151d3a 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java
@@ -70,7 +70,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest {
public void testAddMetrics() throws Exception {
ObjectStore objStore = new ObjectStore();
objStore.setConf(metaStore.getConf());
- objStore.deleteRuntimeStats(0);
+ objStore.deleteReplicationMetrics(0);
ReplicationMetricList replicationMetricList = new ReplicationMetricList();
List<ReplicationMetrics> replicationMetrics = new ArrayList<>();
replicationMetrics.add(createReplicationMetric("repl1", 1L));
@@ -128,7 +128,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest {
public void testUpdateMetrics() throws Exception {
ObjectStore objStore = new ObjectStore();
objStore.setConf(metaStore.getConf());
- objStore.deleteRuntimeStats(0);
+ objStore.deleteReplicationMetrics(0);
ReplicationMetricList replicationMetricList = new ReplicationMetricList();
List<ReplicationMetrics> replicationMetrics = new ArrayList<>();
replicationMetrics.add(createReplicationMetric("repl1", 1L));
@@ -193,7 +193,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest {
public void testGetMetricsByScheduleId() throws Exception {
ObjectStore objStore = new ObjectStore();
objStore.setConf(metaStore.getConf());
- objStore.deleteRuntimeStats(0);
+ objStore.deleteReplicationMetrics(0);
ReplicationMetricList replicationMetricList = new ReplicationMetricList();
List<ReplicationMetrics> replicationMetrics = new ArrayList<>();
replicationMetrics.add(createReplicationMetric("repl1", 1L));
@@ -244,6 +244,45 @@ public class TestReplicationMetrics extends MetaStoreClientTest {
}
+ @Test
+ public void testDeleteMetrics() throws Exception {
+ ObjectStore objStore = new ObjectStore();
+ objStore.setConf(metaStore.getConf());
+ objStore.deleteReplicationMetrics(0);
+ ReplicationMetricList replicationMetricList = new ReplicationMetricList();
+ List<ReplicationMetrics> replicationMetrics = new ArrayList<>();
+ replicationMetrics.add(createReplicationMetric("repl1", 1L));
+ replicationMetrics.add(createReplicationMetric("repl1", 2L));
+ replicationMetricList.setReplicationMetricList(replicationMetrics);
+ objStore.addReplicationMetrics(replicationMetricList);
+ Thread.sleep(2000);
+ replicationMetrics = new ArrayList<>();
+ replicationMetrics.add(createReplicationMetric("repl1", 3L));
+ replicationMetricList.setReplicationMetricList(replicationMetrics);
+ objStore.addReplicationMetrics(replicationMetricList);
+ Thread.sleep(500);
+
+ GetReplicationMetricsRequest getReplicationMetricsRequest = new GetReplicationMetricsRequest();
+ getReplicationMetricsRequest.setPolicy("repl1");
+ ReplicationMetricList actualList = client.getReplicationMetrics(getReplicationMetricsRequest);
+ assertEquals(3, actualList.getReplicationMetricListSize());
+ //delete older metrics
+ objStore.deleteReplicationMetrics(2);
+
+ getReplicationMetricsRequest = new GetReplicationMetricsRequest();
+ getReplicationMetricsRequest.setPolicy("repl1");
+ actualList = client.getReplicationMetrics(getReplicationMetricsRequest);
+ assertEquals(1, actualList.getReplicationMetricListSize());
+ List<ReplicationMetrics> actualMetrics = actualList.getReplicationMetricList();
+ ReplicationMetrics actualMetric0 = actualMetrics.get(0);
+ assertEquals("repl1", actualMetric0.getPolicy());
+ assertEquals(3L, actualMetric0.getScheduledExecutionId());
+ assertEquals(1, actualMetric0.getDumpExecutionId());
+ assertEquals("metadata", actualMetric0.getMetadata());
+ assertEquals("progress", actualMetric0.getProgress());
+
+ }
+
private ReplicationMetrics createReplicationMetric(String policyName, Long scheduleId) {
ReplicationMetrics replicationMetrics = new ReplicationMetrics();
replicationMetrics.setPolicy(policyName);