You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2018/06/01 02:12:56 UTC
asterixdb git commit: [NO ISSUE][STO] Skip deleting unallocated
memory component
Repository: asterixdb
Updated Branches:
refs/heads/master 5a34ce7c7 -> bc7e16e05
[NO ISSUE][STO] Skip deleting unallocated memory component
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- This change fixes the component delete logic
- It first ensures that memory is allocated for the index
- Then it checks whether the memory component is to be deleted
- This is important since there might be cases where primary
index has the memory allocated but not the secondary and
without allocating secondary and deleting it, we could end
up with memory components with different ids.
Change-Id: I0c6c7968830f3c9241bd036c0a330be1400349b4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2668
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/bc7e16e0
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/bc7e16e0
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/bc7e16e0
Branch: refs/heads/master
Commit: bc7e16e0580336442622c82404755a3b65472c4b
Parents: 5a34ce7
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Thu May 31 09:46:58 2018 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Thu May 31 19:12:24 2018 -0700
----------------------------------------------------------------------
.../context/PrimaryIndexOperationTracker.java | 3 +-
.../storage/am/lsm/common/impls/LSMHarness.java | 39 ++++++++++++--------
2 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc7e16e0/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 0aa0d2b..a1d31d5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -132,7 +132,8 @@ public class PrimaryIndexOperationTracker extends BaseOperationTracker implement
for (ILSMIndex lsmIndex : indexes) {
if (lsmIndex.isPrimaryIndex()) {
if (lsmIndex.isCurrentMutableComponentEmpty()) {
- LOGGER.info("Primary index on dataset {} and partition {} is empty... skipping flush");
+ LOGGER.info("Primary index on dataset {} and partition {} is empty... skipping flush",
+ dsInfo.getDatasetID(), partition);
return;
}
primaryLsmIndex = lsmIndex;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc7e16e0/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 7a87a13..4d840b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -483,10 +483,10 @@ public class LSMHarness implements ILSMHarness {
public ILSMIOOperation scheduleFlush(ILSMIndexOperationContext ctx) throws HyracksDataException {
ILSMIOOperation flush;
LOGGER.info("Flush is being scheduled on {}", lsmIndex);
+ if (!lsmIndex.isMemoryComponentsAllocated()) {
+ lsmIndex.allocateMemoryComponents();
+ }
synchronized (opTracker) {
- if (!lsmIndex.isMemoryComponentsAllocated()) {
- lsmIndex.allocateMemoryComponents();
- }
try {
flush = lsmIndex.createFlushOperation(ctx);
} finally {
@@ -738,32 +738,36 @@ public class LSMHarness implements ILSMHarness {
@Override
public void deleteComponents(ILSMIndexOperationContext ctx, Predicate<ILSMComponent> predicate)
throws HyracksDataException {
- boolean deleteMemoryComponent;
ILSMIOOperation ioOperation = null;
+ // We need to always start the component delete from current memory component.
+ // This will ensure Primary and secondary component id still matches after component delete
+ if (!lsmIndex.isMemoryComponentsAllocated()) {
+ lsmIndex.allocateMemoryComponents();
+ }
synchronized (opTracker) {
waitForFlushesAndMerges();
// We always start with the memory component
ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent();
- deleteMemoryComponent = predicate.test(memComponent);
- if (deleteMemoryComponent) {
+ if (predicate.test(memComponent)) {
// schedule a delete for flushed component
ctx.reset();
ctx.setOperation(IndexOperation.DELETE_COMPONENTS);
ioOperation = scheduleFlush(ctx);
+ } else {
+ // since we're not deleting the memory component, we can't delete any previous component
+ return;
}
}
// Here, we are releasing the opTracker to allow other operations:
// (searches, delete flush we will schedule, delete merge we will schedule).
- if (deleteMemoryComponent) {
- try {
- ioOperation.sync();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw HyracksDataException.create(e);
- }
- if (ioOperation.getStatus() == LSMIOOperationStatus.FAILURE) {
- throw HyracksDataException.create(ioOperation.getFailure());
- }
+ try {
+ ioOperation.sync();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw HyracksDataException.create(e);
+ }
+ if (ioOperation.getStatus() == LSMIOOperationStatus.FAILURE) {
+ throw HyracksDataException.create(ioOperation.getFailure());
}
ctx.reset();
ctx.setOperation(IndexOperation.DELETE_COMPONENTS);
@@ -774,6 +778,9 @@ public class LSMHarness implements ILSMHarness {
for (ILSMDiskComponent component : diskComponents) {
if (predicate.test(component)) {
ctx.getComponentsToBeMerged().add(component);
+ } else {
+ // Can't delete older components when newer one is still there
+ break;
}
}
if (ctx.getComponentsToBeMerged().isEmpty()) {