You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/09/30 01:23:37 UTC
asterixdb git commit: Fixes for ASTERIXDB-1636
Repository: asterixdb
Updated Branches:
refs/heads/master 102ef590b -> 2685b60d9
Fixes for ASTERIXDB-1636
The index of the tuple field for filters from SecondaryIndexOperationsHelper and AqlMetadataProvider
differed. The one in AqlMetadataProvider was wrong, as it was attempting to take into account the
presence of a partitioning field in the incoming tuple, which is not there in the case of an
insert/upsert.
There was also an issue where on merge, for components with a filter page but no min/max, the merge
would fail. I fixed this by skipping over null entries while getting the min/max of merging components.
Finally, there was a very silly error in LSMComponentFilterManager which was causing the filter page
to appear as blank, because the page was being pinned with the wrong argument. That is also fixed.
Change-Id: Ib4bc413fcda9a5c98ae57f94e1c8a68fe9aacda3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1205
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Taewoo Kim <wa...@yahoo.com>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2685b60d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2685b60d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2685b60d
Branch: refs/heads/master
Commit: 2685b60d9e03a515fcc5260a3dd4399740f5ec40
Parents: 102ef59
Author: Ian Maxon <im...@apache.org>
Authored: Thu Sep 29 16:38:28 2016 -0700
Committer: Ian Maxon <im...@apache.org>
Committed: Thu Sep 29 18:15:53 2016 -0700
----------------------------------------------------------------------
.../metadata/declared/AqlMetadataProvider.java | 6 ++---
.../common/impls/LSMComponentFilterManager.java | 25 +++++++++-----------
.../invertedindex/impls/LSMInvertedIndex.java | 10 ++++++--
3 files changed, 22 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 29087ba..e05aa25 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -1833,7 +1833,6 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
// One token (+ optional partitioning field) + primary keys: [token,
// number of token, PK]
int numKeys = primaryKeys.size() + secondaryKeys.size();
- int numTokenKeyPairFields = (!isPartitioned) ? 1 + primaryKeys.size() : 2 + primaryKeys.size();
int numFilterFields = DatasetUtils.getFilterField(dataset) == null ? 0 : 1;
// generate field permutations
@@ -1960,8 +1959,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
filterFieldsForNonBulkLoadOps = new int[numFilterFields];
- filterFieldsForNonBulkLoadOps[0] = numTokenKeyPairFields;
- invertedIndexFieldsForNonBulkLoadOps = new int[numTokenKeyPairFields];
+ //for non-bulk-loads, there is only <SK,PK,F> in the incoming tuples
+ filterFieldsForNonBulkLoadOps[0] = numKeys;
+ invertedIndexFieldsForNonBulkLoadOps = new int[numKeys];
for (int k = 0; k < invertedIndexFieldsForNonBulkLoadOps.length; k++) {
invertedIndexFieldsForNonBulkLoadOps[k] = k;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
index d0d7e69..93225cf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
@@ -56,36 +56,33 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager {
}
@Override
- public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex ) throws HyracksDataException {
+ public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
IMetaDataPageManager treeMetaManager = treeIndex.getMetaManager();
ICachedPage filterPage = null;
int componentFilterPageId = treeMetaManager.getFilterPageId();
boolean appendOnly = false;
int fileId = treeIndex.getFileId();
- if(componentFilterPageId == LinkedMetaDataPageManager.NO_FILTER_IN_PLACE){//in-place mode, no filter page yet
+ if (componentFilterPageId == LinkedMetaDataPageManager.NO_FILTER_IN_PLACE) { //in-place mode, no filter page yet
ITreeIndexMetaDataFrame metadataFrame = treeIndex.getMetaManager().getMetaDataFrameFactory().createFrame();
int metaPageId = treeMetaManager.getFirstMetadataPage();
ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metaPageId), false);
metadataPage.acquireWriteLatch();
- try{
+ try {
metadataFrame.setPage(metadataPage);
componentFilterPageId = treeIndex.getMetaManager().getFreePage(metadataFrame);
metadataFrame.setLSMComponentFilterPageId(componentFilterPageId);
- }
- finally{
+ } finally {
metadataPage.releaseWriteLatch(true);
bufferCache.unpin(metadataPage);
}
- }
- else if (componentFilterPageId <= LinkedMetaDataPageManager.NO_FILTER_APPEND_ONLY){
+ } else if (componentFilterPageId <= LinkedMetaDataPageManager.NO_FILTER_APPEND_ONLY) {
appendOnly = true;
filterPage = treeMetaManager.getFilterPage();
- if(filterPage == null){
+ if (filterPage == null) {
treeMetaManager.setFilterPage(bufferCache.confiscatePage(IBufferCache.INVALID_DPID));
filterPage = treeMetaManager.getFilterPage();
}
- }
- else{// in place, not a new filter page
+ } else { // in place, not a new filter page
filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId), true);
}
@@ -102,11 +99,10 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager {
}
} finally {
- if(!appendOnly){
+ if (!appendOnly) {
bufferCache.unpin(filterPage);
filterPage.releaseWriteLatch(true);
- }
- else{
+ } else {
filterPage.releaseWriteLatch(false);
}
}
@@ -122,7 +118,8 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager {
if (componentFilterPageId < 0)
return false;
- ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId), true);
+ ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId),
+ false);
filterPage.acquireReadLatch();
try {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 204e874..bf6e9b5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -645,8 +645,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
- filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
+ ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
+ if (min != null) {
+ filterTuples.add(min);
+ }
+ if (max != null) {
+ filterTuples.add(max);
+ }
}
filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
filterManager.writeFilterInfo(component.getLSMComponentFilter(),