You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bh...@apache.org on 2019/09/16 17:05:32 UTC
[impala] 01/03: IMPALA-7975(continued): Improve supportability of
the automatic invalidate feature.
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1a00227e6aaffd99ffcce30b4beda5eb5f0d2fb1
Author: Anurag Mantripragada <an...@gmail.com>
AuthorDate: Wed Aug 28 14:05:23 2019 -0700
IMPALA-7975(continued): Improve supportability of the automatic
invalidate feature.
This work was done by Sharanitha Harish.
This commit adds support to display the metric last-synced-event-id
as /metrics#events page whereas previously it was displayed
only on the /events page.
Testing:
Added tests toMetaStoreEventsProcessorTest to verify:
1. If the metric updated as it should when new events were processed.
2. If the metric was not set when the event processor was not active.
Change-Id: Iadab4a6cd3ad3c6c1b7931747ac55cd58ce3bb3d
Reviewed-on: http://gerrit.cloudera.org:8080/14200
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/event-metrics.cc | 8 ++++++++
be/src/util/event-metrics.h | 6 ++++++
common/thrift/JniCatalog.thrift | 3 +++
common/thrift/metrics.json | 10 ++++++++++
.../apache/impala/catalog/events/MetastoreEventsProcessor.java | 3 ++-
.../impala/catalog/events/MetastoreEventsProcessorTest.java | 4 ++++
6 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/be/src/util/event-metrics.cc b/be/src/util/event-metrics.cc
index 61151c3..451cbcd 100644
--- a/be/src/util/event-metrics.cc
+++ b/be/src/util/event-metrics.cc
@@ -37,6 +37,8 @@ string MetastoreEventMetrics::EVENTS_RECEIVED_5MIN_METRIC_NAME =
"events-processor.events-received-5min-rate";
string MetastoreEventMetrics::EVENTS_RECEIVED_15MIN_METRIC_NAME =
"events-processor.events-received-15min-rate";
+string MetastoreEventMetrics::LAST_SYNCED_EVENT_ID_METRIC_NAME =
+ "events-processor.last-synced-event-id";
IntCounter* MetastoreEventMetrics::NUM_EVENTS_RECEIVED_COUNTER = nullptr;
IntCounter* MetastoreEventMetrics::NUM_EVENTS_SKIPPED_COUNTER = nullptr;
@@ -49,6 +51,7 @@ StringProperty* MetastoreEventMetrics::EVENT_PROCESSOR_STATUS = nullptr;
DoubleGauge* MetastoreEventMetrics::EVENTS_RECEIVED_1MIN_RATE = nullptr;
DoubleGauge* MetastoreEventMetrics::EVENTS_RECEIVED_5MIN_RATE = nullptr;
DoubleGauge* MetastoreEventMetrics::EVENTS_RECEIVED_15MIN_RATE = nullptr;
+IntCounter* MetastoreEventMetrics::LAST_SYNCED_EVENT_ID = nullptr;
// Initialize all the metrics for the events metric group
void MetastoreEventMetrics::InitMetastoreEventMetrics(MetricGroup* metric_group) {
@@ -73,6 +76,8 @@ void MetastoreEventMetrics::InitMetastoreEventMetrics(MetricGroup* metric_group)
event_metrics->AddDoubleGauge(EVENTS_RECEIVED_5MIN_METRIC_NAME, 0.0);
EVENTS_RECEIVED_15MIN_RATE =
event_metrics->AddDoubleGauge(EVENTS_RECEIVED_15MIN_METRIC_NAME, 0.0);
+ LAST_SYNCED_EVENT_ID =
+ event_metrics->AddCounter(LAST_SYNCED_EVENT_ID_METRIC_NAME, 0);
}
void MetastoreEventMetrics::refresh(TEventProcessorMetrics* response) {
@@ -104,5 +109,8 @@ void MetastoreEventMetrics::refresh(TEventProcessorMetrics* response) {
if (response->__isset.events_received_15min_rate) {
EVENTS_RECEIVED_15MIN_RATE->SetValue(response->events_received_15min_rate);
}
+ if(response->__isset.last_synced_event_id){
+ LAST_SYNCED_EVENT_ID->SetValue(response->last_synced_event_id);
+ }
}
} // namespace impala
\ No newline at end of file
diff --git a/be/src/util/event-metrics.h b/be/src/util/event-metrics.h
index 359ebb8..ffbfa66 100644
--- a/be/src/util/event-metrics.h
+++ b/be/src/util/event-metrics.h
@@ -55,6 +55,9 @@ class MetastoreEventMetrics {
/// EWMA of number of events received in last 15 min
static DoubleGauge* EVENTS_RECEIVED_15MIN_RATE;
+ /// Last metastore event id that the catalog server synced to.
+ static IntCounter* LAST_SYNCED_EVENT_ID;
+
private:
/// Following metric names must match with the key in metrics.json
@@ -81,6 +84,9 @@ class MetastoreEventMetrics {
/// metric name for EWMA of number of events in last 15 min
static string EVENTS_RECEIVED_15MIN_METRIC_NAME;
+
+ /// Metric name for last metastore event id that the catalog server synced to.
+ static string LAST_SYNCED_EVENT_ID_METRIC_NAME;
};
} // namespace impala
diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift
index 8b376b8..40b40a1 100644
--- a/common/thrift/JniCatalog.thrift
+++ b/common/thrift/JniCatalog.thrift
@@ -781,6 +781,9 @@ struct TEventProcessorMetrics {
// Average number events skipped in a polling interval
9: optional double events_skipped_per_poll_mean
+
+ // Last metastore event id that the catalog server synced to
+ 10: optional i64 last_synced_event_id
}
// Response to GetCatalogServerMetrics() call.
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index ed01f67..2d96daf 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -2473,6 +2473,16 @@
"key": "events-processor.events-received-15min-rate"
},
{
+ "description": "Last metastore event id that the catalog server processed and synced to",
+ "contexts": [
+ "CATALOGSERVER"
+ ],
+ "label": "Last Synced Event Id",
+ "units": "NONE",
+ "kind" : "COUNTER",
+ "key" : "events-processor.last-synced-event-id"
+ },
+ {
"description": "Total number of executor groups that have at least one executor",
"contexts": [
"IMPALAD"
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
index e0888f8..72e77e3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
@@ -199,7 +199,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
public static final String EVENTS_SKIPPED_METRIC = "events-skipped";
// name of the event processor status metric
public static final String STATUS_METRIC = "status";
- // last sycned event id
+ // last synced event id
public static final String LAST_SYNCED_ID_METRIC = "last-synced-event-id";
// metric name which counts the number of self-events which are skipped
public static final String NUMBER_OF_SELF_EVENTS = "self-events-skipped";
@@ -525,6 +525,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
TEventProcessorMetrics eventProcessorMetrics = new TEventProcessorMetrics();
EventProcessorStatus currentStatus = getStatus();
eventProcessorMetrics.setStatus(currentStatus.toString());
+ eventProcessorMetrics.setLast_synced_event_id(getLastSyncedEventId());
if (currentStatus != EventProcessorStatus.ACTIVE) return eventProcessorMetrics;
long eventsReceived = metrics_.getMeter(EVENTS_RECEIVED_METRIC).getCount();
diff --git a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index f9d712e..b496d5c 100644
--- a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -1463,6 +1463,7 @@ public class MetastoreEventsProcessorTest {
TEventProcessorMetrics responseBefore = eventsProcessor_.getEventProcessorMetrics();
long numEventsReceivedBefore = responseBefore.getEvents_received();
long numEventsSkippedBefore = responseBefore.getEvents_skipped();
+ long lastEventSyncId = responseBefore.getLast_synced_event_id();
final String testTblName = "testEventProcessorMetrics";
// event 1
createDatabase(TEST_DB_NAME, null);
@@ -1495,6 +1496,7 @@ public class MetastoreEventsProcessorTest {
TEventProcessorMetricsSummaryResponse summaryResponse =
catalog_.getEventProcessorSummary();
assertNotNull(summaryResponse);
+ assertTrue(response.getLast_synced_event_id() > lastEventSyncId);
}
/**
@@ -1518,6 +1520,8 @@ public class MetastoreEventsProcessorTest {
TEventProcessorMetricsSummaryResponse summaryResponse =
eventsProcessor_.getEventProcessorSummary();
assertNotNull(summaryResponse);
+ // Last synced id must be set even when event processor is not active.
+ assertTrue(response.isSetLast_synced_event_id());
} finally {
// reset the state of event process once the test completes
eventsProcessor_.start();