You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2019/06/18 23:50:48 UTC
[incubator-pinot] 01/01: Always download realtime segments on
non-winners
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch always_download_realtime_segments
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0990f83c4d22fdea2d160921311d7bd584714450
Author: Neha Pawar <np...@linkedin.com>
AuthorDate: Tue Jun 18 16:50:18 2019 -0700
Always download realtime segments on non-winners
---
.../manager/realtime/LLRealtimeSegmentDataManager.java | 15 +++++++++++----
pom.xml | 2 +-
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index 7fcbd9c..f5c91e6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -547,14 +547,17 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
_state = State.DISCARDED;
break;
case KEEP:
- _state = State.RETAINING;
+ // Temporary solution. We want to force non-winner servers to always download.
+ _state = State.DISCARDED;
+
+ /*_state = State.RETAINING;
success = buildSegmentAndReplace();
if (success) {
_state = State.RETAINED;
} else {
// Could not build segment for some reason. We can only download it.
_state = State.ERROR;
- }
+ }*/
break;
case COMMIT:
_state = State.COMMITTING;
@@ -911,7 +914,11 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
case CATCHING_UP:
case HOLDING:
case INITIAL_CONSUMING:
- // Allow to catch up upto final offset, and then replace.
+ // Temporary solution. We want to force non-winner servers to always download.
+ segmentLogger.info("State {}. Downloading to replace", _state.toString());
+ downloadSegmentAndReplace(llcMetadata);
+
+ /*// Allow to catch up upto final offset, and then replace.
if (_currentOffset > endOffset) {
// We moved ahead of the offset that is committed in ZK.
segmentLogger.warn("Current offset {} ahead of the offset in zk {}. Downloading to replace", _currentOffset,
@@ -932,7 +939,7 @@ public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
segmentLogger.info("Could not catch up to offset (current = {}). Downloading to replace", _currentOffset);
downloadSegmentAndReplace(llcMetadata);
}
- }
+ }*/
break;
default:
segmentLogger.info("Downloading to replace segment while in state {}", _state.toString());
diff --git a/pom.xml b/pom.xml
index 930fb6b..bfdcdac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,7 +118,7 @@
<!-- Configuration for unit/integration tests section 1 of 3 (properties) ENDS HERE.-->
<avro.version>1.7.6</avro.version>
<parquet.version>1.8.0</parquet.version>
- <helix.version>0.8.2</helix.version>
+ <helix.version>0.8.4</helix.version>
<zkclient.version>0.7</zkclient.version>
<jackson.version>2.9.8</jackson.version>
<async-http-client.version>1.9.21</async-http-client.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org