[impala] 01/03: IMPALA-11644: updateLatestEventId should handle cases of empty events

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit a52721e62761957753f482124c1e76f453d22b1d
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Sun Oct 9 10:37:03 2022 +0800

    IMPALA-11644: updateLatestEventId should handle cases of empty events
    
    IMPALA-11490 adds a catalog metric for the latest event id in HMS. The
    fetched events could be empty if there are no more events in the past 24
    hours, since the retention duration for notification events in HMS is 24
    hours by default. This case is not handled so the thread for updating
    the latestEventId metric will keep throwing a NoSuchElementException
    until there are new events generated.
    
    This patch handles the case to avoid exceptions, also sets the initial
    value of latestEventId to 0 which is the returned value of
    getCurrentNotificationEventId() when there are no events in HMS.
    
    Tests
    - Clean up the notification events in HMS by truncating the
      NOTIFICATION_SEQUENCE and NOTIFICATION_LOG tables in the underlying
      PostgreSQL. Then launch HMS and Impala. Verified the exception
      disappears.
    - Add a breakpoint in updateLatestEventId() to stop before fetching the
      events and after getting the current event id. Clean up the
      notification events in HMS. Then resume catalogd. Verified no
      exceptions are thrown.
    
    Change-Id: I0f207fff1ff59376e30afdc3cd074c950a1c3ddb
    Reviewed-on: http://gerrit.cloudera.org:8080/19112
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../apache/impala/catalog/events/MetastoreEventsProcessor.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
index 264caab4e..7c2d6d368 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
@@ -26,6 +26,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -497,7 +498,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
 
   // The event id and eventTime of the latest event in HMS. Only used in metrics to show
   // how far we are lagging behind.
-  private final AtomicLong latestEventId_ = new AtomicLong(-1);
+  private final AtomicLong latestEventId_ = new AtomicLong(0);
   private final AtomicLong latestEventTimeMs_ = new AtomicLong(0);
 
   // The duration in nanoseconds of the processing of the last event batch.
@@ -888,7 +889,10 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
       eventRequest.setMaxEvents(1);
       NotificationEventResponse response = MetastoreShim
           .getNextNotification(msClient.getHiveClient(), eventRequest);
-      NotificationEvent event = response.getEventsIterator().next();
+      Iterator<NotificationEvent> eventIter = response.getEventsIterator();
+      // Events could be empty if they are just cleaned up.
+      if (!eventIter.hasNext()) return;
+      NotificationEvent event = eventIter.next();
       Preconditions.checkState(event.getEventId() == currentEventId);
       LOG.info("Latest event in HMS: id={}, time={}", currentEventId,
           event.getEventTime());