You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2016/04/07 16:59:38 UTC
[03/50] [abbrv] incubator-asterixdb git commit: Exclude Temporary
Indexes From Replication
Exclude Temporary Indexes From Replication
- Exclude non-durable LSM indexes from replication.
- Stop heartbeat task after NC has stopped to avoid false failures detection.
Change-Id: Icce91a203e04cb068a7a5aa541720bbd0289eacb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/777
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/08aa051a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/08aa051a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/08aa051a
Branch: refs/heads/master
Commit: 08aa051a34d5c2b757cc0e0b2e018725e164d0f7
Parents: b2e9d08
Author: Murtadha Hubail <mh...@uci.edu>
Authored: Sat Apr 2 20:55:44 2016 -0700
Committer: Murtadha Hubail <hu...@gmail.com>
Committed: Sun Apr 3 00:18:32 2016 -0700
----------------------------------------------------------------------
.../control/nc/NodeControllerService.java | 9 +++++--
.../storage/am/lsm/common/api/ILSMIndex.java | 6 +++++
.../am/lsm/common/impls/AbstractLSMIndex.java | 25 +++++++++++---------
.../storage/am/lsm/common/impls/LSMHarness.java | 3 ++-
4 files changed, 29 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 7b5758c..598d6db 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -338,12 +338,17 @@ public class NodeControllerService implements IControllerService {
}
partitionManager.close();
datasetPartitionManager.close();
- heartbeatTask.cancel();
netManager.stop();
datasetNetworkManager.stop();
queue.stop();
- if (ncAppEntryPoint != null)
+ if (ncAppEntryPoint != null) {
ncAppEntryPoint.stop();
+ }
+ /**
+ * Stop heartbeat after NC has stopped to avoid false node failure detection
+ * on CC if an NC takes a long time to stop.
+ */
+ heartbeatTask.cancel();
LOGGER.log(Level.INFO, "Stopped NodeControllerService");
shuttedDown = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index c1cef2d..11b933d 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -39,6 +39,7 @@ public interface ILSMIndex extends IIndex {
public void deactivate(boolean flushOnExit) throws HyracksDataException;
+ @Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException;
@@ -51,4 +52,9 @@ public interface ILSMIndex extends IIndex {
public List<ILSMComponent> getImmutableComponents();
public boolean isPrimaryIndex();
+
+ /**
+ * @return true if the index is durable. Otherwise false.
+ */
+ public boolean isDurable();
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 441dda1..440ad31 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -33,7 +33,6 @@ import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionTy
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -48,8 +47,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public abstract class AbstractLSMIndex implements ILSMIndexInternal {
@@ -80,11 +77,10 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
protected boolean memoryComponentsAllocated = false;
public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
- int[] filterFields, boolean durable) {
+ ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) {
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
@@ -149,8 +145,8 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
}
protected void markAsValidInternal(IBufferCache bufferCache, BloomFilter filter) throws HyracksDataException {
- if(durable){
- bufferCache.force(filter.getFileId(),true);
+ if (durable) {
+ bufferCache.force(filter.getFileId(), true);
}
}
@@ -208,7 +204,7 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
return diskBufferCache;
}
- public boolean isEmptyIndex() throws HyracksDataException {
+ public boolean isEmptyIndex() {
boolean isModified = false;
for (ILSMComponent c : memoryComponents) {
AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) c;
@@ -288,9 +284,16 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
}
}
+ @Override
public abstract void allocateMemoryComponents() throws HyracksDataException;
+ @Override
public boolean isMemoryComponentsAllocated() {
return memoryComponentsAllocated;
}
+
+ @Override
+ public boolean isDurable() {
+ return durable;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 21b0d8a..0224c5c 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -63,7 +63,8 @@ public class LSMHarness implements ILSMHarness {
this.opTracker = opTracker;
this.mergePolicy = mergePolicy;
fullMergeIsRequested = new AtomicBoolean();
- this.replicationEnabled = replicationEnabled;
+ //only durable indexes are replicated
+ this.replicationEnabled = replicationEnabled && lsmIndex.isDurable();
if (replicationEnabled) {
this.componentsToBeReplicated = new ArrayList<ILSMComponent>();
}