You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2020/07/31 17:57:45 UTC
[incubator-pinot] branch emit-server-metric-when-retry-fails
created (now 743e28f)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch emit-server-metric-when-retry-fails
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 743e28f Emit server metric when retry attempts exceeded
This branch includes the following new commits:
new 743e28f Emit server metric when retry attempts exceeded
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Emit server metric when retry attempts
exceeded
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch emit-server-metric-when-retry-fails
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 743e28f9d6a8ee618bf8b3b3b3740820bc985f3f
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Jul 31 10:57:14 2020 -0700
Emit server metric when retry attempts exceeded
---
.../apache/pinot/common/metrics/ServerMeter.java | 1 +
.../starter/helix/SegmentFetcherAndLoader.java | 27 ++++++++++++++--------
2 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
index 0a8a30e..06ea32f 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
@@ -62,6 +62,7 @@ public enum ServerMeter implements AbstractMetrics.Meter {
RELOAD_FAILURES("segments", false),
REFRESH_FAILURES("segments", false),
UNTAR_FAILURES("segments", false),
+ RETRY_EXCEEDED_FAILURES("segments", false),
// Netty connection metrics
NETTY_CONNECTION_BYTES_RECEIVED("nettyConnection", true),
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
index 56c78dc..341772d 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/SegmentFetcherAndLoader.java
@@ -43,6 +43,7 @@ import org.apache.pinot.spi.crypt.PinotCrypter;
import org.apache.pinot.spi.crypt.PinotCrypterFactory;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
+import org.apache.pinot.spi.utils.retry.AttemptsExceededException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,17 +197,23 @@ public class SegmentFetcherAndLoader {
File tempTarFile = new File(tempDir, segmentName + TAR_GZ_SUFFIX);
File tempSegmentDir = new File(tempDir, segmentName);
try {
- SegmentFetcherFactory.fetchSegmentToLocal(uri, tempDownloadFile);
- if (crypter != null) {
- crypter.decrypt(tempDownloadFile, tempTarFile);
- } else {
- tempTarFile = tempDownloadFile;
+ try {
+ SegmentFetcherFactory.fetchSegmentToLocal(uri, tempDownloadFile);
+ if (crypter != null) {
+ crypter.decrypt(tempDownloadFile, tempTarFile);
+ } else {
+ tempTarFile = tempDownloadFile;
+ }
+ LOGGER.info("Downloaded tarred segment: {} for table: {} from: {} to: {}, file length: {}", segmentName,
+ tableName, uri, tempTarFile, tempTarFile.length());
+ } catch (AttemptsExceededException e) {
+ LOGGER.error("Attempts exceeded when downloading segment: {} for table: {} from: {} to: {}", segmentName,
+ tableName, uri, tempTarFile);
+ _serverMetrics.addMeteredTableValue(tableName, ServerMeter.RETRY_EXCEEDED_FAILURES, 1L);
+ Utils.rethrowException(e);
+ return null;
}
- LOGGER
- .info("Downloaded tarred segment: {} for table: {} from: {} to: {}, file length: {}", segmentName, tableName,
- uri, tempTarFile, tempTarFile.length());
-
try {
// If an exception is thrown when untarring, it means the tar file is broken OR not found after the retry.
// Thus, there's no need to retry again.
@@ -220,6 +227,8 @@ public class SegmentFetcherAndLoader {
LOGGER.info("Successfully downloaded segment: {} for table: {} to: {}", segmentName, tableName, indexDir);
return indexDir.getAbsolutePath();
} catch (Exception e) {
+ LOGGER.error("Exception when untarring segment: {} for table: {} from {} to {}", segmentName, tableName,
+ tempTarFile, tempSegmentDir);
_serverMetrics.addMeteredTableValue(tableName, ServerMeter.UNTAR_FAILURES, 1L);
Utils.rethrowException(e);
return null;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org