You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2022/10/19 19:53:21 UTC
[accumulo] branch main updated: Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4917b3032d Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032)
4917b3032d is described below
commit 4917b3032deb097adf1573086c0c3c3f11265f0a
Author: Keith Turner <kt...@apache.org>
AuthorDate: Wed Oct 19 20:53:16 2022 +0100
Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032)
---
.../apache/accumulo/core/iterators/Combiner.java | 3 ++-
.../apache/accumulo/core/util/ShutdownUtil.java | 24 ++++++++++++++++++++++
.../main/java/org/apache/accumulo/fate/Fate.java | 15 +-------------
.../apache/accumulo/tserver/tablet/Scanner.java | 11 +++++++---
.../apache/accumulo/tserver/tablet/TabletBase.java | 11 +++++++++-
5 files changed, 45 insertions(+), 19 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
index 23bbfeade9..fedb25ca90 100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
import static java.util.concurrent.TimeUnit.HOURS;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -130,7 +131,7 @@ public abstract class Combiner extends WrappingIterator implements OptionDescrib
source.next();
hasNext = _hasNext();
} catch (IOException e) {
- throw new RuntimeException(e);
+ throw new UncheckedIOException(e);
}
return topValue;
}
diff --git a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
index b1cd2f0afb..0fbf90fbda 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.core.util;
+import java.io.IOException;
+
public class ShutdownUtil {
/**
@@ -34,4 +36,26 @@ public class ShutdownUtil {
return false;
}
+ public static boolean isIOException(Throwable e) {
+ if (e == null)
+ return false;
+
+ if (e instanceof IOException)
+ return true;
+
+ for (Throwable suppressed : e.getSuppressed())
+ if (isIOException(suppressed))
+ return true;
+
+ return isIOException(e.getCause());
+ }
+
+ /**
+ * @return true if there is a possibility that the exception was caused by the hadoop shutdown
+ * hook closing the hadoop file system objects, otherwise false
+ */
+ public static boolean wasCausedByHadoopShutdown(Exception e) {
+ return isShutdownInProgress() && isIOException(e);
+ }
+
}
diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java b/core/src/main/java/org/apache/accumulo/fate/Fate.java
index d45f039e28..2a98b6e15b 100644
--- a/core/src/main/java/org/apache/accumulo/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.fate;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.accumulo.core.util.ShutdownUtil.isIOException;
import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED;
import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS;
import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.IN_PROGRESS;
@@ -135,20 +136,6 @@ public class Fate<T> {
}
}
- private boolean isIOException(Throwable e) {
- if (e == null)
- return false;
-
- if (e instanceof IOException)
- return true;
-
- for (Throwable suppressed : e.getSuppressed())
- if (isIOException(suppressed))
- return true;
-
- return isIOException(e.getCause());
- }
-
/**
* The Hadoop Filesystem registers a java shutdown hook that closes the file system. This can
* cause threads to get spurious IOException. If this happens, instead of failing a FATE
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
index 08e2f20afa..2fd917ecda 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
@@ -125,16 +125,21 @@ public class Scanner {
else
throw iie;
} catch (IOException ioe) {
- if (ShutdownUtil.isShutdownInProgress()) {
+ if (ShutdownUtil.wasCausedByHadoopShutdown(ioe)) {
log.debug("IOException while shutdown in progress ", ioe);
- throw new TabletClosedException(ioe); // assume IOException was caused by execution of HDFS
- // shutdown hook
+ throw new TabletClosedException(ioe); // this was possibly caused by Hadoop shutdown hook,
+ // so make the client retry
}
sawException = true;
dataSource.close(true);
throw ioe;
} catch (RuntimeException re) {
+ if (ShutdownUtil.wasCausedByHadoopShutdown(re)) {
+ log.debug("RuntimeException while shutdown in progress ", re);
+ throw new TabletClosedException(re); // this was possibly caused by Hadoop shutdown hook, so
+ // make the client retry
+ }
sawException = true;
throw re;
} finally {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index 031adc0619..ce8cddc348 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -410,7 +410,7 @@ public abstract class TabletBase {
entriesAdded);
tabletClosed = true;
} catch (IOException ioe) {
- if (ShutdownUtil.isShutdownInProgress()) {
+ if (ShutdownUtil.wasCausedByHadoopShutdown(ioe)) {
// assume HDFS shutdown hook caused this exception
log.debug("IOException while shutdown in progress", ioe);
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range,
@@ -431,6 +431,15 @@ public abstract class TabletBase {
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range,
entriesAdded);
tabletClosed = true;
+ } catch (RuntimeException re) {
+ if (ShutdownUtil.wasCausedByHadoopShutdown(re)) {
+ log.debug("RuntimeException while shutdown in progress", re);
+ handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range,
+ entriesAdded);
+ tabletClosed = true;
+ } else {
+ throw re;
+ }
}
}