You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2020/07/31 17:57:46 UTC
[incubator-pinot] 01/01: Emit server metric when retry attempts
exceeded
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch emit-server-metric-when-retry-fails
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 743e28f9d6a8ee618bf8b3b3b3740820bc985f3f
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Jul 31 10:57:14 2020 -0700
Emit server metric when retry attempts exceeded
---
.../apache/pinot/common/metrics/ServerMeter.java | 1 +
.../starter/helix/SegmentFetcherAndLoader.java | 27 ++++++++++++++--------
2 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
index 0a8a30e..06ea32f 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
@@ -62,6 +62,7 @@ public enum ServerMeter implements AbstractMetrics.Meter {
RELOAD_FAILURES("segments", false),
REFRESH_FAILURES("segments", false),
UNTAR_FAILURES("segments", false),
+ RETRY_EXCEEDED_FAILURES("segments", false),
// Netty connection metrics
NETTY_CONNECTION_BYTES_RECEIVED("nettyConnection", true),
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
index 56c78dc..341772d 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
@@ -43,6 +43,7 @@ import org.apache.pinot.spi.crypt.PinotCrypter;
import org.apache.pinot.spi.crypt.PinotCrypterFactory;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
+import org.apache.pinot.spi.utils.retry.AttemptsExceededException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,17 +197,23 @@ public class SegmentFetcherAndLoader {
File tempTarFile = new File(tempDir, segmentName + TAR_GZ_SUFFIX);
File tempSegmentDir = new File(tempDir, segmentName);
try {
- SegmentFetcherFactory.fetchSegmentToLocal(uri, tempDownloadFile);
- if (crypter != null) {
- crypter.decrypt(tempDownloadFile, tempTarFile);
- } else {
- tempTarFile = tempDownloadFile;
+ try {
+ SegmentFetcherFactory.fetchSegmentToLocal(uri, tempDownloadFile);
+ if (crypter != null) {
+ crypter.decrypt(tempDownloadFile, tempTarFile);
+ } else {
+ tempTarFile = tempDownloadFile;
+ }
+ LOGGER.info("Downloaded tarred segment: {} for table: {} from: {} to: {}, file length: {}", segmentName,
+ tableName, uri, tempTarFile, tempTarFile.length());
+ } catch (AttemptsExceededException e) {
+ LOGGER.error("Attempts exceeded when downloading segment: {} for table: {} from: {} to: {}", segmentName,
+ tableName, uri, tempTarFile);
+ _serverMetrics.addMeteredTableValue(tableName, ServerMeter.RETRY_EXCEEDED_FAILURES, 1L);
+ Utils.rethrowException(e);
+ return null;
}
- LOGGER
- .info("Downloaded tarred segment: {} for table: {} from: {} to: {}, file length: {}", segmentName, tableName,
- uri, tempTarFile, tempTarFile.length());
-
try {
// If an exception is thrown when untarring, it means the tar file is broken OR not found after the retry.
// Thus, there's no need to retry again.
@@ -220,6 +227,8 @@ public class SegmentFetcherAndLoader {
LOGGER.info("Successfully downloaded segment: {} for table: {} to: {}", segmentName, tableName, indexDir);
return indexDir.getAbsolutePath();
} catch (Exception e) {
+ LOGGER.error("Exception when untarring segment: {} for table: {} from {} to {}", segmentName, tableName,
+ tempTarFile, tempSegmentDir);
_serverMetrics.addMeteredTableValue(tableName, ServerMeter.UNTAR_FAILURES, 1L);
Utils.rethrowException(e);
return null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org