You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wa...@apache.org on 2018/02/19 17:07:48 UTC
[1/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Repository: asterixdb
Updated Branches:
refs/heads/master c587da107 -> afe0d3d99
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
index 0ee1e59..d487bf5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/pom.xml
@@ -73,6 +73,11 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
index d316d03..ee88411 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
@@ -25,8 +25,8 @@ import org.apache.hyracks.storage.am.common.datagen.TupleGenerator;
import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext.InvertedIndexType;
-import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils;
+import org.apache.hyracks.storage.common.IIndex;
import org.junit.Test;
public abstract class AbstractInvertedIndexDeleteTest extends AbstractInvertedIndexTest {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
index 1a0fd87..d05a984 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
@@ -31,8 +31,8 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.search.EditDistanceSearch
import org.apache.hyracks.storage.am.lsm.invertedindex.search.JaccardSearchModifier;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext.InvertedIndexType;
-import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils;
+import org.apache.hyracks.storage.common.IIndex;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
index 757c9d8..a420ba9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexTest.java
@@ -29,8 +29,8 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveSearchM
import org.apache.hyracks.storage.am.lsm.invertedindex.search.JaccardSearchModifier;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext.InvertedIndexType;
-import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils;
+import org.apache.hyracks.storage.common.IIndex;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
index ca06f6b..78b1658 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/multithread/LSMInvertedIndexTestWorker.java
@@ -23,15 +23,25 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.dataflow.std.buffermanager.DeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.FramePoolBackedFrameBufferManager;
+import org.apache.hyracks.dataflow.std.buffermanager.IDeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.AbstractIndexTestWorker;
+import org.apache.hyracks.storage.am.common.TestOperationCallback;
import org.apache.hyracks.storage.am.common.TestOperationSelector;
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexAccessor;
@@ -39,7 +49,9 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveSearchM
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.JaccardSearchModifier;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext;
import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexCursor;
public class LSMInvertedIndexTestWorker extends AbstractIndexTestWorker {
@@ -54,6 +66,18 @@ public class LSMInvertedIndexTestWorker extends AbstractIndexTestWorker {
public LSMInvertedIndexTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, IIndex index,
int numBatches) throws HyracksDataException {
super(dataGen, opSelector, index, numBatches);
+ // Dummy hyracks task context for the test purpose only
+ IHyracksTaskContext ctx = new HyracksTaskTestContext();
+ // Intermediate and final search result will use this buffer manager to get frames.
+ IDeallocatableFramePool framePool = new DeallocatableFramePool(ctx,
+ AccessMethodTestsConfig.LSM_INVINDEX_SEARCH_FRAME_LIMIT * ctx.getInitialFrameSize());;
+ ISimpleFrameBufferManager bufferManagerForSearch = new FramePoolBackedFrameBufferManager(framePool);;
+ // Keep the buffer manager in the hyracks context so that the search process can get it via the context.
+ TaskUtil.put(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, bufferManagerForSearch, ctx);
+ IIndexAccessParameters iap =
+ new IndexAccessParameters(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ iap.getParameters().put(HyracksConstants.HYRACKS_TASK_CONTEXT, ctx);
+ indexAccessor = index.createAccessor(iap);
invIndex = (LSMInvertedIndex) index;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleTest.java
index 4e5ccb9..445ae9b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleTest.java
@@ -23,15 +23,11 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Random;
-import junit.framework.Assert;
-
-import org.junit.Test;
-
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAccessor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAppender;
+import org.junit.Test;
+
+import junit.framework.Assert;
public class FixedSizeFrameTupleTest {
@@ -60,7 +56,7 @@ public class FixedSizeFrameTupleTest {
ArrayList<Integer> check = new ArrayList<Integer>();
- ftapp.reset(buffer, true);
+ ftapp.reset(buffer);
while (frameHasSpace) {
int val = rnd.nextInt();
frameHasSpace = ftapp.append(val);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index 7997327..ae8713c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -27,22 +27,34 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.utils.SerdeUtils;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.dataflow.std.buffermanager.DeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.FramePoolBackedFrameBufferManager;
+import org.apache.hyracks.dataflow.std.buffermanager.IDeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.btree.OrderedIndexTestContext;
import org.apache.hyracks.storage.am.common.CheckTuple;
+import org.apache.hyracks.storage.am.common.TestOperationCallback;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext;
import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
@@ -72,6 +84,18 @@ public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
this.tokenizerFactory = tokenizerFactory;
this.invIndexType = invIndexType;
this.indexTupleIter = indexTupleIter;
+ // Dummy hyracks task context for the test purpose only
+ IHyracksTaskContext ctx = new HyracksTaskTestContext();
+ // Intermediate and final search result will use this buffer manager to get frames.
+ IDeallocatableFramePool framePool = new DeallocatableFramePool(ctx,
+ AccessMethodTestsConfig.LSM_INVINDEX_SEARCH_FRAME_LIMIT * ctx.getInitialFrameSize());;
+ ISimpleFrameBufferManager bufferManagerForSearch = new FramePoolBackedFrameBufferManager(framePool);;
+ // Keep the buffer manager in the hyracks context so that the search process can get it via the context.
+ TaskUtil.put(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, bufferManagerForSearch, ctx);
+ IIndexAccessParameters iap =
+ new IndexAccessParameters(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ iap.getParameters().put(HyracksConstants.HYRACKS_TASK_CONTEXT, ctx);
+ indexAccessor = index.createAccessor(iap);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index 3423f70..db3bb50 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -25,19 +25,37 @@ import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import org.apache.hyracks.api.context.IHyracksJobletContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.TaskAttemptId;
+import org.apache.hyracks.api.dataflow.state.IStateObject;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
+import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.job.JobFlag;
+import org.apache.hyracks.api.job.profiling.IStatsCollector;
+import org.apache.hyracks.api.job.profiling.counters.ICounterContext;
+import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.util.GrowableArray;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -46,6 +64,11 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
+import org.apache.hyracks.dataflow.std.buffermanager.DeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.FramePoolBackedFrameBufferManager;
+import org.apache.hyracks.dataflow.std.buffermanager.IDeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.btree.OrderedIndexTestUtils;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.CheckTuple;
@@ -54,12 +77,15 @@ import org.apache.hyracks.storage.am.common.datagen.IFieldValueGenerator;
import org.apache.hyracks.storage.am.common.datagen.PersonNameFieldValueGenerator;
import org.apache.hyracks.storage.am.common.datagen.SortedIntegerFieldValueGenerator;
import org.apache.hyracks.storage.am.common.datagen.TupleGenerator;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
@@ -73,6 +99,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8Strin
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8NGramTokenFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestContext.InvertedIndexType;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -323,7 +350,7 @@ public class LSMInvertedIndexTestUtils {
ArrayTupleBuilder searchKeyBuilder = new ArrayTupleBuilder(tokenFieldCount);
ArrayTupleReference searchKey = new ArrayTupleReference();
// Cursor over inverted list from actual index.
- IInvertedListCursor actualInvListCursor = invIndexAccessor.createInvertedListCursor();
+ InvertedListCursor actualInvListCursor = invIndexAccessor.createInvertedListCursor();
// Helpers for generating a serialized inverted-list element from a CheckTuple from the expected index.
ArrayTupleBuilder expectedBuilder = new ArrayTupleBuilder(fieldSerdes.length);
@@ -361,7 +388,8 @@ public class LSMInvertedIndexTestUtils {
}
// Compare inverted-list elements.
int count = 0;
- actualInvListCursor.pinPages();
+ actualInvListCursor.prepareLoadPages();
+ actualInvListCursor.loadPages();
try {
while (actualInvListCursor.hasNext() && expectedInvListIter.hasNext()) {
actualInvListCursor.next();
@@ -376,7 +404,8 @@ public class LSMInvertedIndexTestUtils {
count++;
}
} finally {
- actualInvListCursor.unpinPages();
+ actualInvListCursor.unloadPages();
+ actualInvListCursor.close();
}
}
}
@@ -492,8 +521,19 @@ public class LSMInvertedIndexTestUtils {
int numDocQueries, int numRandomQueries, IInvertedIndexSearchModifier searchModifier, int[] scanCountArray)
throws IOException, HyracksDataException {
IInvertedIndex invIndex = testCtx.invIndex;
- IInvertedIndexAccessor accessor =
- (IInvertedIndexAccessor) invIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+
+ // Dummy hyracks task context for the test purpose only
+ IHyracksTaskContext ctx = new HyracksTaskTestContext();
+ // Intermediate and final search result will use this buffer manager to get frames.
+ IDeallocatableFramePool framePool = new DeallocatableFramePool(ctx,
+ AccessMethodTestsConfig.LSM_INVINDEX_SEARCH_FRAME_LIMIT * ctx.getInitialFrameSize());;
+ ISimpleFrameBufferManager bufferManagerForSearch = new FramePoolBackedFrameBufferManager(framePool);;
+ // Keep the buffer manager in the hyracks context so that the search process can get it via the context.
+ TaskUtil.put(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, bufferManagerForSearch, ctx);
+ IIndexAccessParameters iap =
+ new IndexAccessParameters(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ iap.getParameters().put(HyracksConstants.HYRACKS_TASK_CONTEXT, ctx);
+ IInvertedIndexAccessor accessor = (IInvertedIndexAccessor) invIndex.createAccessor(iap);
IBinaryTokenizer tokenizer = testCtx.getTokenizerFactory().createTokenizer();
InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizer, searchModifier);
List<ITupleReference> documentCorpus = testCtx.getDocumentCorpus();
@@ -584,4 +624,128 @@ public class LSMInvertedIndexTestUtils {
}
}
}
+
+ // This is just a dummy hyracks context for allocating frames for temporary
+ // results during inverted index searches for the test purposes only.
+ public static class HyracksTaskTestContext implements IHyracksTaskContext {
+ private final int FRAME_SIZE = AccessMethodTestsConfig.LSM_INVINDEX_HYRACKS_FRAME_SIZE;
+ private Object sharedObject;
+
+ @Override
+ public int getInitialFrameSize() {
+ return FRAME_SIZE;
+ }
+
+ @Override
+ public IIOManager getIoManager() {
+ return null;
+ }
+
+ @Override
+ public ByteBuffer allocateFrame() {
+ return ByteBuffer.allocate(FRAME_SIZE);
+ }
+
+ @Override
+ public ByteBuffer allocateFrame(int bytes) throws HyracksDataException {
+ return ByteBuffer.allocate(bytes);
+ }
+
+ @Override
+ public ByteBuffer reallocateFrame(ByteBuffer bytes, int newSizeInBytes, boolean copyOldData)
+ throws HyracksDataException {
+ throw new HyracksDataException("TODO");
+ }
+
+ @Override
+ public void deallocateFrames(int bytes) {
+ // no-op
+ }
+
+ @Override
+ public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
+ public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
+ public void registerDeallocatable(IDeallocatable deallocatable) {
+ // no-op
+ }
+
+ @Override
+ public void setStateObject(IStateObject taskState) {
+ // no-op
+ }
+
+ @Override
+ public IStateObject getStateObject(Object id) {
+ return null;
+ }
+
+ @Override
+ public IHyracksJobletContext getJobletContext() {
+ return null;
+ }
+
+ @Override
+ public TaskAttemptId getTaskAttemptId() {
+ return null;
+ }
+
+ @Override
+ public ICounterContext getCounterContext() {
+ return null;
+ }
+
+ @Override
+ public ExecutorService getExecutorService() {
+ return null;
+ }
+
+ @Override
+ public IDatasetPartitionManager getDatasetPartitionManager() {
+ return null;
+ }
+
+ @Override
+ public void sendApplicationMessageToCC(Serializable message, DeploymentId deploymentId) throws Exception {
+ // no-op
+ }
+
+ @Override
+ public void sendApplicationMessageToCC(byte[] message, DeploymentId deploymentId) throws Exception {
+ // no-op
+ }
+
+ @Override
+ public void setSharedObject(Object object) {
+ this.sharedObject = object;
+ }
+
+ @Override
+ public Object getSharedObject() {
+ return sharedObject;
+ }
+
+ @Override
+ public byte[] getJobParameter(byte[] name, int start, int length) throws HyracksException {
+ return null;
+ }
+
+ @Override
+ public Set<JobFlag> getJobFlags() {
+ return null;
+ }
+
+ @Override
+ public IStatsCollector getStatsCollector() {
+ return null;
+ }
+ }
+
}
[6/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/data/csv/fragile_sample.csv
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/data/csv/fragile_sample.csv b/asterixdb/asterix-app/data/csv/fragile_sample.csv
new file mode 100644
index 0000000..4a14083
--- /dev/null
+++ b/asterixdb/asterix-app/data/csv/fragile_sample.csv
@@ -0,0 +1,20000 @@
+1,1,20130417,1,9:58:59,84,0.714285714
+2,1,20130417,1,9:59:00,77,0.779220779
+3,1,20130417,1,9:59:00,77,0.779220779
+4,1,20130417,1,9:59:01,83,0.722891566
+5,1,20130417,1,9:59:02,81,0.740740741
+6,1,20130417,1,9:59:03,78,0.769230769
+7,1,20130417,1,9:59:03,78,0.769230769
+8,1,20130417,1,9:59:04,81,0.740740741
+9,1,20130417,1,9:59:05,79,0.759493671
+10,1,20130417,1,9:59:06,78,0.769230769
+11,1,20130417,1,9:59:06,78,0.769230769
+12,1,20130417,1,9:59:07,82,0.731707317
+13,1,20130417,1,9:59:08,81,0.740740741
+14,1,20130417,1,9:59:09,82,0.731707317
+15,1,20130417,1,9:59:09,87,0.689655172
+16,1,20130417,1,9:59:10,87,0.689655172
+17,1,20130417,1,9:59:11,88,0.681818182
+18,1,20130417,1,9:59:12,89,0.674157303
+19,1,20130417,1,9:59:12,91,0.659340659
+20,1,20130417,1,9:59:13,87,0.689655172
+21,1,20130417,1,9:59:14,80,0.75
+22,1,20130417,1,9:59:14,81,0.740740741
+23,1,20130417,1,9:59:15,81,0.740740741
+24,1,20130417,1,9:59:16,78,0.769230769
+25,1,20130417,1,9:59:17,82,0.731707317
+26,1,20130417,1,9:59:17,86,0.697674419
+27,1,20130417,1,9:59:18,89,0.674157303
+28,1,20130417,1,9:59:19,91,0.659340659
+29,1,20130417,1,9:59:19,93,0.64516129
+30,1,20130417,1,9:59:20,94,0.638297872
+31,1,20130417,1,9:59:21,94,0.638297872
+32,1,20130417,1,9:59:21,94,0.638297872
+33,1,20130417,1,9:59:22,90,0.666666667
+34,1,20130417,1,9:59:23,85,0.705882353
+35,1,20130417,1,9:59:23,83,0.722891566
+36,1,20130417,1,9:59:24,72,0.833333333
+37,1,20130417,1,9:59:25,76,0.789473684
+38,1,20130417,1,9:59:26,76,0.789473684
+39,1,20130417,1,9:59:26,73,0.821917808
+40,1,20130417,1,9:59:27,80,0.75
+41,1,20130417,1,9:59:28,80,0.75
+42,1,20130417,1,9:59:29,84,0.714285714
+43,1,20130417,1,9:59:29,88,0.681818182
+44,1,20130417,1,9:59:30,92,0.652173913
+45,1,20130417,1,9:59:31,93,0.64516129
+46,1,20130417,1,9:59:31,94,0.638297872
+47,1,20130417,1,9:59:32,95,0.631578947
+48,1,20130417,1,9:59:33,98,0.612244898
+49,1,20130417,1,9:59:33,92,0.652173913
+50,1,20130417,1,9:59:34,98,0.612244898
+51,1,20130417,1,9:59:35,98,0.612244898
+52,1,20130417,1,9:59:35,96,0.625
+53,1,20130417,1,9:59:36,99,0.606060606
+54,1,20130417,1,9:59:36,101,0.594059406
+55,1,20130417,1,9:59:37,99,0.606060606
+56,1,20130417,1,9:59:38,99,0.606060606
+57,1,20130417,1,9:59:38,100,0.6
+58,1,20130417,1,9:59:39,102,0.588235294
+59,1,20130417,1,9:59:39,99,0.606060606
+60,1,20130417,1,9:59:40,99,0.606060606
+61,1,20130417,1,9:59:41,96,0.625
+62,1,20130417,1,9:59:41,83,0.722891566
+63,1,20130417,1,9:59:42,80,0.75
+64,1,20130417,1,9:59:43,84,0.714285714
+65,1,20130417,1,9:59:43,81,0.740740741
+66,1,20130417,1,9:59:44,78,0.769230769
+67,1,20130417,1,9:59:45,83,0.722891566
+68,1,20130417,1,9:59:46,83,0.722891566
+69,1,20130417,1,9:59:46,83,0.722891566
+70,1,20130417,1,9:59:47,87,0.689655172
+71,1,20130417,1,9:59:48,91,0.659340659
+72,1,20130417,1,9:59:48,85,0.705882353
+73,1,20130417,1,9:59:49,86,0.697674419
+74,1,20130417,1,9:59:50,88,0.681818182
+75,1,20130417,1,9:59:51,83,0.722891566
+76,1,20130417,1,9:59:51,84,0.714285714
+77,1,20130417,1,9:59:52,84,0.714285714
+78,1,20130417,1,9:59:53,82,0.731707317
+79,1,20130417,1,9:59:53,87,0.689655172
+80,1,20130417,1,9:59:54,83,0.722891566
+81,1,20130417,1,9:59:55,83,0.722891566
+82,1,20130417,1,9:59:56,85,0.705882353
+83,1,20130417,1,9:59:56,82,0.731707317
+84,1,20130417,1,9:59:57,88,0.681818182
+85,1,20130417,1,9:59:58,92,0.652173913
+86,1,20130417,1,9:59:58,90,0.666666667
+87,1,20130417,1,9:59:59,92,0.652173913
+88,1,20130417,1,10:00:00,96,0.625
+89,1,20130417,1,10:00:00,94,0.638297872
+90,1,20130417,1,10:00:01,93,0.64516129
+91,1,20130417,1,10:00:02,93,0.64516129
+92,1,20130417,1,10:00:02,96,0.625
+93,1,20130417,1,10:00:03,91,0.659340659
+94,1,20130417,1,10:00:03,92,0.652173913
+95,1,20130417,1,10:00:04,94,0.638297872
+96,1,20130417,1,10:00:05,83,0.722891566
+97,1,20130417,1,10:00:05,87,0.689655172
+98,1,20130417,1,10:00:06,91,0.659340659
+99,1,20130417,1,10:00:07,87,0.689655172
+100,1,20130417,1,10:00:08,92,0.652173913
+101,1,20130417,1,10:00:08,86,0.697674419
+102,1,20130417,1,10:00:09,86,0.697674419
+103,1,20130417,1,10:00:10,85,0.705882353
+104,1,20130417,1,10:00:10,91,0.659340659
+105,1,20130417,1,10:00:11,89,0.674157303
+106,1,20130417,1,10:00:12,89,0.674157303
+107,1,20130417,1,10:00:12,88,0.681818182
+108,1,20130417,1,10:00:13,91,0.659340659
+109,1,20130417,1,10:00:14,87,0.689655172
+110,1,20130417,1,10:00:14,90,0.666666667
+111,1,20130417,1,10:00:15,91,0.659340659
+112,1,20130417,1,10:00:16,88,0.681818182
+113,1,20130417,1,10:00:16,84,0.714285714
+114,1,20130417,1,10:00:17,82,0.731707317
+115,1,20130417,1,10:00:18,85,0.705882353
+116,1,20130417,1,10:00:18,84,0.714285714
+117,1,20130417,1,10:00:19,83,0.722891566
+118,1,20130417,1,10:00:20,86,0.697674419
+119,1,20130417,1,10:00:21,86,0.697674419
+120,1,20130417,1,10:00:21,86,0.697674419
+121,1,20130417,1,10:00:22,83,0.722891566
+122,1,20130417,1,10:00:23,83,0.722891566
+123,1,20130417,1,10:00:23,72,0.833333333
+124,1,20130417,1,10:00:24,70,0.857142857
+125,1,20130417,1,10:00:25,76,0.789473684
+126,1,20130417,1,10:00:26,74,0.810810811
+127,1,20130417,1,10:00:27,74,0.810810811
+128,1,20130417,1,10:00:28,76,0.789473684
+129,1,20130417,1,10:00:28,82,0.731707317
+130,1,20130417,1,10:00:29,86,0.697674419
+131,1,20130417,1,10:00:30,86,0.697674419
+132,1,20130417,1,10:00:30,88,0.681818182
+133,1,20130417,1,10:00:31,89,0.674157303
+134,1,20130417,1,10:00:32,91,0.659340659
+135,1,20130417,1,10:00:32,89,0.674157303
+136,1,20130417,1,10:00:33,85,0.705882353
+137,1,20130417,1,10:00:34,84,0.714285714
+138,1,20130417,1,10:00:35,88,0.681818182
+139,1,20130417,1,10:00:35,87,0.689655172
+140,1,20130417,1,10:00:36,92,0.652173913
+141,1,20130417,1,10:00:37,91,0.659340659
+142,1,20130417,1,10:00:37,86,0.697674419
+143,1,20130417,1,10:00:38,87,0.689655172
+144,1,20130417,1,10:00:39,88,0.681818182
+145,1,20130417,1,10:00:39,85,0.705882353
+146,1,20130417,1,10:00:40,80,0.75
+147,1,20130417,1,10:00:41,82,0.731707317
+148,1,20130417,1,10:00:42,85,0.705882353
+149,1,20130417,1,10:00:42,81,0.740740741
+150,1,20130417,1,10:00:43,81,0.740740741
+151,1,20130417,1,10:00:44,81,0.740740741
+152,1,20130417,1,10:00:44,86,0.697674419
+153,1,20130417,1,10:00:45,84,0.714285714
+154,1,20130417,1,10:00:46,88,0.681818182
+155,1,20130417,1,10:00:47,93,0.64516129
+156,1,20130417,1,10:00:47,93,0.64516129
+157,1,20130417,1,10:00:48,88,0.681818182
+158,1,20130417,1,10:00:49,82,0.731707317
+159,1,20130417,1,10:00:49,82,0.731707317
+160,1,20130417,1,10:00:50,77,0.779220779
+161,1,20130417,1,10:00:51,75,0.8
+162,1,20130417,1,10:00:52,77,0.779220779
+163,1,20130417,1,10:00:52,82,0.731707317
+164,1,20130417,1,10:00:53,82,0.731707317
+165,1,20130417,1,10:00:54,86,0.697674419
+166,1,20130417,1,10:00:54,85,0.705882353
+167,1,20130417,1,10:00:55,86,0.697674419
+168,1,20130417,1,10:00:56,87,0.689655172
+169,1,20130417,1,10:00:57,83,0.722891566
+170,1,20130417,1,10:00:57,82,0.731707317
+171,1,20130417,1,10:00:58,83,0.722891566
+172,1,20130417,1,10:00:59,81,0.740740741
+173,1,20130417,1,10:01:00,83,0.722891566
+174,1,20130417,1,10:01:00,81,0.740740741
+175,1,20130417,1,10:01:01,85,0.705882353
+176,1,20130417,1,10:01:02,80,0.75
+177,1,20130417,1,10:01:02,82,0.731707317
+178,1,20130417,1,10:01:03,87,0.689655172
+179,1,20130417,1,10:01:04,87,0.689655172
+180,1,20130417,1,10:01:05,81,0.740740741
+181,1,20130417,1,10:01:05,80,0.75
+182,1,20130417,1,10:01:06,82,0.731707317
+183,1,20130417,1,10:01:07,81,0.740740741
+184,1,20130417,1,10:01:08,77,0.779220779
+185,1,20130417,1,10:01:08,73,0.821917808
+186,1,20130417,1,10:01:09,77,0.779220779
+187,1,20130417,1,10:01:10,75,0.8
+188,1,20130417,1,10:01:11,74,0.810810811
+189,1,20130417,1,10:01:12,75,0.8
+190,1,20130417,1,10:01:12,80,0.75
+191,1,20130417,1,10:01:13,76,0.789473684
+192,1,20130417,1,10:01:14,76,0.789473684
+193,1,20130417,1,10:01:15,78,0.769230769
+194,1,20130417,1,10:01:15,80,0.75
+195,1,20130417,1,10:01:16,83,0.722891566
+196,1,20130417,1,10:01:17,85,0.705882353
+197,1,20130417,1,10:01:18,91,0.659340659
+198,1,20130417,1,10:01:18,92,0.652173913
+199,1,20130417,1,10:01:19,81,0.740740741
+200,1,20130417,1,10:01:20,91,0.659340659
+201,1,20130417,1,10:01:20,80,0.75
+202,1,20130417,1,10:01:21,80,0.75
+203,1,20130417,1,10:01:22,83,0.722891566
+204,1,20130417,1,10:01:23,88,0.681818182
+205,1,20130417,1,10:01:23,88,0.681818182
+206,1,20130417,1,10:01:24,86,0.697674419
+207,1,20130417,1,10:01:25,86,0.697674419
+208,1,20130417,1,10:01:25,87,0.689655172
+209,1,20130417,1,10:01:26,88,0.681818182
+210,1,20130417,1,10:01:27,88,0.681818182
+211,1,20130417,1,10:01:27,82,0.731707317
+212,1,20130417,1,10:01:28,78,0.769230769
+213,1,20130417,1,10:01:29,80,0.75
+214,1,20130417,1,10:01:30,77,0.779220779
+215,1,20130417,1,10:01:30,76,0.789473684
+216,1,20130417,1,10:01:31,78,0.769230769
+217,1,20130417,1,10:01:32,79,0.759493671
+218,1,20130417,1,10:01:33,72,0.833333333
+219,1,20130417,1,10:01:34,75,0.8
+220,1,20130417,1,10:01:34,75,0.8
+221,1,20130417,1,10:01:35,69,0.869565217
+222,1,20130417,1,10:01:36,72,0.833333333
+223,1,20130417,1,10:01:37,79,0.759493671
+224,1,20130417,1,10:01:38,71,0.845070423
+225,1,20130417,1,10:01:38,74,0.810810811
+226,1,20130417,1,10:01:39,84,0.714285714
+227,1,20130417,1,10:01:40,82,0.731707317
+228,1,20130417,1,10:01:41,82,0.731707317
+229,1,20130417,1,10:01:41,84,0.714285714
+230,1,20130417,1,10:01:42,89,0.674157303
+231,1,20130417,1,10:01:43,85,0.705882353
+232,1,20130417,1,10:01:44,84,0.714285714
+233,1,20130417,1,10:01:44,85,0.705882353
+234,1,20130417,1,10:01:45,83,0.722891566
+235,1,20130417,1,10:01:46,82,0.731707317
+236,1,20130417,1,10:01:46,92,0.652173913
+237,1,20130417,1,10:01:47,84,0.714285714
+238,1,20130417,1,10:01:48,83,0.722891566
+239,1,20130417,1,10:01:48,86,0.697674419
+240,1,20130417,1,10:01:49,91,0.659340659
+241,1,20130417,1,10:01:50,90,0.666666667
+242,1,20130417,1,10:01:51,92,0.652173913
+243,1,20130417,1,10:01:51,93,0.64516129
+244,1,20130417,1,10:01:52,93,0.64516129
+245,1,20130417,1,10:01:52,92,0.652173913
+246,1,20130417,1,10:01:53,93,0.64516129
+247,1,20130417,1,10:01:54,96,0.625
+248,1,20130417,1,10:01:54,96,0.625
+249,1,20130417,1,10:01:55,94,0.638297872
+250,1,20130417,1,10:01:56,93,0.64516129
+251,1,20130417,1,10:01:56,96,0.625
+252,1,20130417,1,10:01:57,86,0.697674419
+253,1,20130417,1,10:01:58,88,0.681818182
+254,1,20130417,1,10:01:58,90,0.666666667
+255,1,20130417,1,10:01:59,86,0.697674419
+256,1,20130417,1,10:02:00,83,0.722891566
+257,1,20130417,1,10:02:00,89,0.674157303
+258,1,20130417,1,10:02:01,89,0.674157303
+259,1,20130417,1,10:02:02,89,0.674157303
+260,1,20130417,1,10:02:02,92,0.652173913
+261,1,20130417,1,10:02:03,87,0.689655172
+262,1,20130417,1,10:02:04,87,0.689655172
+263,1,20130417,1,10:02:04,93,0.64516129
+264,1,20130417,1,10:02:05,97,0.618556701
+265,1,20130417,1,10:02:06,96,0.625
+266,1,20130417,1,10:02:06,101,0.594059406
+267,1,20130417,1,10:02:07,100,0.6
+268,1,20130417,1,10:02:08,100,0.6
+269,1,20130417,1,10:02:08,101,0.594059406
+270,1,20130417,1,10:02:09,99,0.606060606
+271,1,20130417,1,10:02:09,98,0.612244898
+272,1,20130417,1,10:02:10,99,0.606060606
+273,1,20130417,1,10:02:11,100,0.6
+274,1,20130417,1,10:02:11,99,0.606060606
+275,1,20130417,1,10:02:12,99,0.606060606
+276,1,20130417,1,10:02:12,97,0.618556701
+277,1,20130417,1,10:02:13,104,0.576923077
+278,1,20130417,1,10:02:14,103,0.582524272
+279,1,20130417,1,10:02:14,105,0.571428571
+280,1,20130417,1,10:02:15,107,0.560747664
+281,1,20130417,1,10:02:15,102,0.588235294
+282,1,20130417,1,10:02:16,98,0.612244898
+283,1,20130417,1,10:02:16,105,0.571428571
+284,1,20130417,1,10:02:17,100,0.6
+285,1,20130417,1,10:02:18,101,0.594059406
+286,1,20130417,1,10:02:18,102,0.588235294
+287,1,20130417,1,10:02:19,105,0.571428571
+288,1,20130417,1,10:02:19,102,0.588235294
+289,1,20130417,1,10:02:20,101,0.594059406
+290,1,20130417,1,10:02:21,104,0.576923077
+291,1,20130417,1,10:02:21,102,0.588235294
+292,1,20130417,1,10:02:22,103,0.582524272
+293,1,20130417,1,10:02:22,104,0.576923077
+294,1,20130417,1,10:02:23,106,0.566037736
+295,1,20130417,1,10:02:23,103,0.582524272
+296,1,20130417,1,10:02:24,101,0.594059406
+297,1,20130417,1,10:02:25,99,0.606060606
+298,1,20130417,1,10:02:25,104,0.576923077
+299,1,20130417,1,10:02:26,99,0.606060606
+300,1,20130417,1,10:02:26,95,0.631578947
+301,1,20130417,1,10:02:27,96,0.625
+302,1,20130417,1,10:02:28,97,0.618556701
+303,1,20130417,1,10:02:28,87,0.689655172
+304,1,20130417,1,10:02:29,85,0.705882353
+305,1,20130417,1,10:02:30,89,0.674157303
+306,1,20130417,1,10:02:30,86,0.697674419
+307,1,20130417,1,10:02:31,88,0.681818182
+308,1,20130417,1,10:02:32,97,0.618556701
+309,1,20130417,1,10:02:32,91,0.659340659
+310,1,20130417,1,10:02:33,91,0.659340659
+311,1,20130417,1,10:02:34,95,0.631578947
+312,1,20130417,1,10:02:34,91,0.659340659
+313,1,20130417,1,10:02:35,90,0.666666667
+314,1,20130417,1,10:02:36,94,0.638297872
+315,1,20130417,1,10:02:36,96,0.625
+316,1,20130417,1,10:02:37,93,0.64516129
+317,1,20130417,1,10:02:38,95,0.631578947
+318,1,20130417,1,10:02:38,94,0.638297872
+319,1,20130417,1,10:02:39,97,0.618556701
+320,1,20130417,1,10:02:39,93,0.64516129
+321,1,20130417,1,10:02:40,92,0.652173913
+322,1,20130417,1,10:02:41,96,0.625
+323,1,20130417,1,10:02:41,95,0.631578947
+324,1,20130417,1,10:02:42,94,0.638297872
+325,1,20130417,1,10:02:43,97,0.618556701
+326,1,20130417,1,10:02:43,94,0.638297872
+327,1,20130417,1,10:02:44,91,0.659340659
+328,1,20130417,1,10:02:45,97,0.618556701
+329,1,20130417,1,10:02:45,91,0.659340659
+330,1,20130417,1,10:02:46,90,0.666666667
+331,1,20130417,1,10:02:46,94,0.638297872
+332,1,20130417,1,10:02:47,93,0.64516129
+333,1,20130417,1,10:02:48,90,0.666666667
+334,1,20130417,1,10:02:48,95,0.631578947
+335,1,20130417,1,10:02:49,96,0.625
+336,1,20130417,1,10:02:50,93,0.64516129
+337,1,20130417,1,10:02:50,100,0.6
+338,1,20130417,1,10:02:51,97,0.618556701
+339,1,20130417,1,10:02:52,99,0.606060606
+340,1,20130417,1,10:02:52,103,0.582524272
+341,1,20130417,1,10:02:53,100,0.6
+342,1,20130417,1,10:02:53,99,0.606060606
+343,1,20130417,1,10:02:54,102,0.588235294
+344,1,20130417,1,10:02:55,100,0.6
+345,1,20130417,1,10:02:55,98,0.612244898
+346,1,20130417,1,10:02:56,99,0.606060606
+347,1,20130417,1,10:02:56,100,0.6
+348,1,20130417,1,10:02:57,95,0.631578947
+349,1,20130417,1,10:02:58,94,0.638297872
+350,1,20130417,1,10:02:58,96,0.625
+351,1,20130417,1,10:02:59,90,0.666666667
+352,1,20130417,1,10:03:00,91,0.659340659
+353,1,20130417,1,10:03:00,96,0.625
+354,1,20130417,1,10:03:01,94,0.638297872
+355,1,20130417,1,10:03:01,94,0.638297872
+356,1,20130417,1,10:03:02,98,0.612244898
+357,1,20130417,1,10:03:03,92,0.652173913
+358,1,20130417,1,10:03:03,91,0.659340659
+359,1,20130417,1,10:03:04,98,0.612244898
+360,1,20130417,1,10:03:05,95,0.631578947
+361,1,20130417,1,10:03:05,94,0.638297872
+362,1,20130417,1,10:03:06,97,0.618556701
+363,1,20130417,1,10:03:07,96,0.625
+364,1,20130417,1,10:03:07,92,0.652173913
+365,1,20130417,1,10:03:08,92,0.652173913
+366,1,20130417,1,10:03:08,94,0.638297872
+367,1,20130417,1,10:03:09,90,0.666666667
+368,1,20130417,1,10:03:10,94,0.638297872
+369,1,20130417,1,10:03:10,92,0.652173913
+370,1,20130417,1,10:03:11,93,0.64516129
+371,1,20130417,1,10:03:12,95,0.631578947
+372,1,20130417,1,10:03:12,93,0.64516129
+373,1,20130417,1,10:03:13,96,0.625
+374,1,20130417,1,10:03:14,97,0.618556701
+375,1,20130417,1,10:03:14,96,0.625
+376,1,20130417,1,10:03:15,97,0.618556701
+377,1,20130417,1,10:03:15,102,0.588235294
+378,1,20130417,1,10:03:16,99,0.606060606
+379,1,20130417,1,10:03:17,99,0.606060606
+380,1,20130417,1,10:03:17,100,0.6
+381,1,20130417,1,10:03:18,99,0.606060606
+382,1,20130417,1,10:03:18,94,0.638297872
+383,1,20130417,1,10:03:19,89,0.674157303
+384,1,20130417,1,10:03:20,95,0.631578947
+385,1,20130417,1,10:03:20,87,0.689655172
+386,1,20130417,1,10:03:21,90,0.666666667
+387,1,20130417,1,10:03:22,94,0.638297872
+388,1,20130417,1,10:03:22,92,0.652173913
+389,1,20130417,1,10:03:23,89,0.674157303
+390,1,20130417,1,10:03:24,94,0.638297872
+391,1,20130417,1,10:03:24,92,0.652173913
+392,1,20130417,1,10:03:25,90,0.666666667
+393,1,20130417,1,10:03:26,94,0.638297872
+394,1,20130417,1,10:03:26,98,0.612244898
+395,1,20130417,1,10:03:27,100,0.6
+396,1,20130417,1,10:03:28,99,0.606060606
+397,1,20130417,1,10:03:28,100,0.6
+398,1,20130417,1,10:03:29,96,0.625
+399,1,20130417,1,10:03:29,97,0.618556701
+400,1,20130417,1,10:03:30,102,0.588235294
+401,1,20130417,1,10:03:31,99,0.606060606
+402,1,20130417,1,10:03:31,96,0.625
+403,1,20130417,1,10:03:32,101,0.594059406
+404,1,20130417,1,10:03:32,102,0.588235294
+405,1,20130417,1,10:03:33,100,0.6
+406,1,20130417,1,10:03:34,94,0.638297872
+407,1,20130417,1,10:03:34,98,0.612244898
+408,1,20130417,1,10:03:35,102,0.588235294
+409,1,20130417,1,10:03:35,96,0.625
+410,1,20130417,1,10:03:36,96,0.625
+411,1,20130417,1,10:03:37,100,0.6
+412,1,20130417,1,10:03:37,102,0.588235294
+413,1,20130417,1,10:03:38,101,0.594059406
+414,1,20130417,1,10:03:38,91,0.659340659
+415,1,20130417,1,10:03:39,93,0.64516129
+416,1,20130417,1,10:03:40,100,0.6
+417,1,20130417,1,10:03:40,99,0.606060606
+418,1,20130417,1,10:03:41,100,0.6
+419,1,20130417,1,10:03:42,101,0.594059406
+420,1,20130417,1,10:03:42,96,0.625
+421,1,20130417,1,10:03:43,96,0.625
+422,1,20130417,1,10:03:43,93,0.64516129
+423,1,20130417,1,10:03:44,86,0.697674419
+424,1,20130417,1,10:03:45,93,0.64516129
+425,1,20130417,1,10:03:45,98,0.612244898
+426,1,20130417,1,10:03:46,104,0.576923077
+427,1,20130417,1,10:03:47,99,0.606060606
+428,1,20130417,1,10:03:47,100,0.6
+429,1,20130417,1,10:03:48,95,0.631578947
+430,1,20130417,1,10:03:48,100,0.6
+431,1,20130417,1,10:03:49,101,0.594059406
+432,1,20130417,1,10:03:50,99,0.606060606
+433,1,20130417,1,10:03:50,102,0.588235294
+434,1,20130417,1,10:03:51,106,0.566037736
+435,1,20130417,1,10:03:51,108,0.555555556
+436,1,20130417,1,10:03:52,109,0.550458716
+437,1,20130417,1,10:03:52,112,0.535714286
+438,1,20130417,1,10:03:53,110,0.545454545
+439,1,20130417,1,10:03:54,104,0.576923077
+440,1,20130417,1,10:03:54,102,0.588235294
+441,1,20130417,1,10:03:55,104,0.576923077
+442,1,20130417,1,10:03:55,107,0.560747664
+443,1,20130417,1,10:03:56,103,0.582524272
+444,1,20130417,1,10:03:56,104,0.576923077
+445,1,20130417,1,10:03:57,103,0.582524272
+446,1,20130417,1,10:03:58,101,0.594059406
+447,1,20130417,1,10:03:58,99,0.606060606
+448,1,20130417,1,10:03:59,103,0.582524272
+449,1,20130417,1,10:03:59,101,0.594059406
+450,1,20130417,1,10:04:00,98,0.612244898
+451,1,20130417,1,10:04:01,100,0.6
+452,1,20130417,1,10:04:01,96,0.625
+453,1,20130417,1,10:04:02,96,0.625
+454,1,20130417,1,10:04:02,98,0.612244898
+455,1,20130417,1,10:04:03,99,0.606060606
+456,1,20130417,1,10:04:04,93,0.64516129
+457,1,20130417,1,10:04:04,98,0.612244898
+458,1,20130417,1,10:04:05,92,0.652173913
+459,1,20130417,1,10:04:06,90,0.666666667
+460,1,20130417,1,10:04:06,97,0.618556701
+461,1,20130417,1,10:04:07,96,0.625
+462,1,20130417,1,10:04:07,94,0.638297872
+463,1,20130417,1,10:04:08,99,0.606060606
+464,1,20130417,1,10:04:09,100,0.6
+465,1,20130417,1,10:04:09,99,0.606060606
+466,1,20130417,1,10:04:10,99,0.606060606
+467,1,20130417,1,10:04:11,101,0.594059406
+468,1,20130417,1,10:04:11,93,0.64516129
+469,1,20130417,1,10:04:12,98,0.612244898
+470,1,20130417,1,10:04:12,102,0.588235294
+471,1,20130417,1,10:04:13,96,0.625
+472,1,20130417,1,10:04:14,91,0.659340659
+473,1,20130417,1,10:04:14,98,0.612244898
+474,1,20130417,1,10:04:15,95,0.631578947
+475,1,20130417,1,10:04:16,91,0.659340659
+476,1,20130417,1,10:04:16,95,0.631578947
+477,1,20130417,1,10:04:17,96,0.625
+478,1,20130417,1,10:04:17,93,0.64516129
+479,1,20130417,1,10:04:18,96,0.625
+480,1,20130417,1,10:04:19,100,0.6
+481,1,20130417,1,10:04:19,100,0.6
+482,1,20130417,1,10:04:20,96,0.625
+483,1,20130417,1,10:04:21,99,0.606060606
+484,1,20130417,1,10:04:21,100,0.6
+485,1,20130417,1,10:04:22,97,0.618556701
+486,1,20130417,1,10:04:22,98,0.612244898
+487,1,20130417,1,10:04:23,101,0.594059406
+488,1,20130417,1,10:04:24,94,0.638297872
+489,1,20130417,1,10:04:24,97,0.618556701
+490,1,20130417,1,10:04:25,99,0.606060606
+491,1,20130417,1,10:04:25,94,0.638297872
+492,1,20130417,1,10:04:26,95,0.631578947
+493,1,20130417,1,10:04:27,99,0.606060606
+494,1,20130417,1,10:04:27,94,0.638297872
+495,1,20130417,1,10:04:28,96,0.625
+496,1,20130417,1,10:04:29,99,0.606060606
+497,1,20130417,1,10:04:29,96,0.625
+498,1,20130417,1,10:04:30,91,0.659340659
+499,1,20130417,1,10:04:30,100,0.6
+500,1,20130417,1,10:04:31,94,0.638297872
+501,1,20130417,1,10:04:32,98,0.612244898
+502,1,20130417,1,10:04:32,102,0.588235294
+503,1,20130417,1,10:04:33,102,0.588235294
+504,1,20130417,1,10:04:33,103,0.582524272
+505,1,20130417,1,10:04:34,104,0.576923077
+506,1,20130417,1,10:04:35,107,0.560747664
+507,1,20130417,1,10:04:35,106,0.566037736
+508,1,20130417,1,10:04:36,97,0.618556701
+509,1,20130417,1,10:04:36,99,0.606060606
+510,1,20130417,1,10:04:37,94,0.638297872
+511,1,20130417,1,10:04:38,99,0.606060606
+512,1,20130417,1,10:04:38,98,0.612244898
+513,1,20130417,1,10:04:39,99,0.606060606
+514,1,20130417,1,10:04:39,99,0.606060606
+515,1,20130417,1,10:04:40,100,0.6
+516,1,20130417,1,10:04:41,97,0.618556701
+517,1,20130417,1,10:04:41,99,0.606060606
+518,1,20130417,1,10:04:42,98,0.612244898
+519,1,20130417,1,10:04:42,98,0.612244898
+520,1,20130417,1,10:04:43,100,0.6
+521,1,20130417,1,10:04:44,99,0.606060606
+522,1,20130417,1,10:04:44,94,0.638297872
+523,1,20130417,1,10:04:45,101,0.594059406
+524,1,20130417,1,10:04:46,98,0.612244898
+525,1,20130417,1,10:04:46,97,0.618556701
+526,1,20130417,1,10:04:47,96,0.625
+527,1,20130417,1,10:04:47,99,0.606060606
+528,1,20130417,1,10:04:48,92,0.652173913
+529,1,20130417,1,10:04:49,91,0.659340659
+530,1,20130417,1,10:04:49,97,0.618556701
+531,1,20130417,1,10:04:50,87,0.689655172
+532,1,20130417,1,10:04:51,87,0.689655172
+533,1,20130417,1,10:04:51,93,0.64516129
+534,1,20130417,1,10:04:52,90,0.666666667
+535,1,20130417,1,10:04:53,90,0.666666667
+536,1,20130417,1,10:04:53,96,0.625
+537,1,20130417,1,10:04:54,95,0.631578947
+538,1,20130417,1,10:04:55,97,0.618556701
+539,1,20130417,1,10:04:55,102,0.588235294
+540,1,20130417,1,10:04:56,103,0.582524272
+541,1,20130417,1,10:04:56,102,0.588235294
+542,1,20130417,1,10:04:57,100,0.6
+543,1,20130417,1,10:04:58,104,0.576923077
+544,1,20130417,1,10:04:58,98,0.612244898
+545,1,20130417,1,10:04:59,96,0.625
+546,1,20130417,1,10:04:59,99,0.606060606
+547,1,20130417,1,10:05:00,90,0.666666667
+548,1,20130417,1,10:05:01,95,0.631578947
+549,1,20130417,1,10:05:01,97,0.618556701
+550,1,20130417,1,10:05:02,92,0.652173913
+551,1,20130417,1,10:05:03,90,0.666666667
+552,1,20130417,1,10:05:03,96,0.625
+553,1,20130417,1,10:05:04,90,0.666666667
+554,1,20130417,1,10:05:04,87,0.689655172
+555,1,20130417,1,10:05:05,96,0.625
+556,1,20130417,1,10:05:06,96,0.625
+557,1,20130417,1,10:05:06,92,0.652173913
+558,1,20130417,1,10:05:07,94,0.638297872
+559,1,20130417,1,10:05:08,99,0.606060606
+560,1,20130417,1,10:05:08,89,0.674157303
+561,1,20130417,1,10:05:09,88,0.681818182
+562,1,20130417,1,10:05:10,96,0.625
+563,1,20130417,1,10:05:10,92,0.652173913
+564,1,20130417,1,10:05:11,90,0.666666667
+565,1,20130417,1,10:05:12,97,0.618556701
+566,1,20130417,1,10:05:12,98,0.612244898
+567,1,20130417,1,10:05:13,89,0.674157303
+568,1,20130417,1,10:05:14,92,0.652173913
+569,1,20130417,1,10:05:14,96,0.625
+570,1,20130417,1,10:05:15,92,0.652173913
+571,1,20130417,1,10:05:15,90,0.666666667
+572,1,20130417,1,10:05:16,91,0.659340659
+573,1,20130417,1,10:05:17,98,0.612244898
+574,1,20130417,1,10:05:17,91,0.659340659
+575,1,20130417,1,10:05:18,94,0.638297872
+576,1,20130417,1,10:05:19,98,0.612244898
+577,1,20130417,1,10:05:19,92,0.652173913
+578,1,20130417,1,10:05:20,91,0.659340659
+579,1,20130417,1,10:05:21,93,0.64516129
+580,1,20130417,1,10:05:21,93,0.64516129
+581,1,20130417,1,10:05:22,89,0.674157303
+582,1,20130417,1,10:05:23,90,0.666666667
+583,1,20130417,1,10:05:23,97,0.618556701
+584,1,20130417,1,10:05:24,90,0.666666667
+585,1,20130417,1,10:05:25,91,0.659340659
+586,1,20130417,1,10:05:25,95,0.631578947
+587,1,20130417,1,10:05:26,91,0.659340659
+588,1,20130417,1,10:05:26,92,0.652173913
+589,1,20130417,1,10:05:27,95,0.631578947
+590,1,20130417,1,10:05:28,92,0.652173913
+591,1,20130417,1,10:05:28,91,0.659340659
+592,1,20130417,1,10:05:29,94,0.638297872
+593,1,20130417,1,10:05:30,96,0.625
+594,1,20130417,1,10:05:30,91,0.659340659
+595,1,20130417,1,10:05:31,96,0.625
+596,1,20130417,1,10:05:32,99,0.606060606
+597,1,20130417,1,10:05:32,98,0.612244898
+598,1,20130417,1,10:05:33,92,0.652173913
+599,1,20130417,1,10:05:33,97,0.618556701
+600,1,20130417,1,10:05:34,98,0.612244898
+601,1,20130417,1,10:05:35,92,0.652173913
+602,1,20130417,1,10:05:35,96,0.625
+603,1,20130417,1,10:05:36,98,0.612244898
+604,1,20130417,1,10:05:37,91,0.659340659
+605,1,20130417,1,10:05:37,93,0.64516129
+606,1,20130417,1,10:05:38,99,0.606060606
+607,1,20130417,1,10:05:39,96,0.625
+608,1,20130417,1,10:05:39,94,0.638297872
+609,1,20130417,1,10:05:40,100,0.6
+610,1,20130417,1,10:05:40,101,0.594059406
+611,1,20130417,1,10:05:41,102,0.588235294
+612,1,20130417,1,10:05:42,98,0.612244898
+613,1,20130417,1,10:05:42,103,0.582524272
+614,1,20130417,1,10:05:43,108,0.555555556
+615,1,20130417,1,10:05:43,101,0.594059406
+616,1,20130417,1,10:05:44,101,0.594059406
+617,1,20130417,1,10:05:45,92,0.652173913
+618,1,20130417,1,10:05:45,94,0.638297872
+619,1,20130417,1,10:05:46,94,0.638297872
+620,1,20130417,1,10:05:46,98,0.612244898
+621,1,20130417,1,10:05:47,103,0.582524272
+622,1,20130417,1,10:05:48,102,0.588235294
+623,1,20130417,1,10:05:48,105,0.571428571
+624,1,20130417,1,10:05:49,100,0.6
+625,1,20130417,1,10:05:49,99,0.606060606
+626,1,20130417,1,10:05:50,98,0.612244898
+627,1,20130417,1,10:05:51,99,0.606060606
+628,1,20130417,1,10:05:51,92,0.652173913
+629,1,20130417,1,10:05:52,90,0.666666667
+630,1,20130417,1,10:05:53,93,0.64516129
+631,1,20130417,1,10:05:53,97,0.618556701
+632,1,20130417,1,10:05:54,91,0.659340659
+633,1,20130417,1,10:05:54,91,0.659340659
+634,1,20130417,1,10:05:55,93,0.64516129
+635,1,20130417,1,10:05:56,98,0.612244898
+636,1,20130417,1,10:05:56,99,0.606060606
+637,1,20130417,1,10:05:57,96,0.625
+638,1,20130417,1,10:05:58,95,0.631578947
+639,1,20130417,1,10:05:58,99,0.606060606
+640,1,20130417,1,10:05:59,90,0.666666667
+641,1,20130417,1,10:06:00,88,0.681818182
+642,1,20130417,1,10:06:00,97,0.618556701
+643,1,20130417,1,10:06:01,92,0.652173913
+644,1,20130417,1,10:06:01,88,0.681818182
+645,1,20130417,1,10:06:02,92,0.652173913
+646,1,20130417,1,10:06:03,99,0.606060606
+647,1,20130417,1,10:06:03,91,0.659340659
+648,1,20130417,1,10:06:04,91,0.659340659
+649,1,20130417,1,10:06:05,96,0.625
+650,1,20130417,1,10:06:05,98,0.612244898
+651,1,20130417,1,10:06:06,96,0.625
+652,1,20130417,1,10:06:07,98,0.612244898
+653,1,20130417,1,10:06:07,94,0.638297872
+654,1,20130417,1,10:06:08,93,0.64516129
+655,1,20130417,1,10:06:08,98,0.612244898
+656,1,20130417,1,10:06:09,92,0.652173913
+657,1,20130417,1,10:06:10,91,0.659340659
+658,1,20130417,1,10:06:10,94,0.638297872
+659,1,20130417,1,10:06:11,88,0.681818182
+660,1,20130417,1,10:06:12,90,0.666666667
+661,1,20130417,1,10:06:12,96,0.625
+662,1,20130417,1,10:06:13,88,0.681818182
+663,1,20130417,1,10:06:14,95,0.631578947
+664,1,20130417,1,10:06:14,96,0.625
+665,1,20130417,1,10:06:15,94,0.638297872
+666,1,20130417,1,10:06:16,90,0.666666667
+667,1,20130417,1,10:06:16,89,0.674157303
+668,1,20130417,1,10:06:17,90,0.666666667
+669,1,20130417,1,10:06:18,85,0.705882353
+670,1,20130417,1,10:06:18,84,0.714285714
+671,1,20130417,1,10:06:19,90,0.666666667
+672,1,20130417,1,10:06:20,83,0.722891566
+673,1,20130417,1,10:06:20,84,0.714285714
+674,1,20130417,1,10:06:21,92,0.652173913
+675,1,20130417,1,10:06:22,88,0.681818182
+676,1,20130417,1,10:06:22,88,0.681818182
+677,1,20130417,1,10:06:23,85,0.705882353
+678,1,20130417,1,10:06:24,89,0.674157303
+679,1,20130417,1,10:06:25,83,0.722891566
+680,1,20130417,1,10:06:25,83,0.722891566
+681,1,20130417,1,10:06:26,89,0.674157303
+682,1,20130417,1,10:06:27,90,0.666666667
+683,1,20130417,1,10:06:27,90,0.666666667
+684,1,20130417,1,10:06:28,94,0.638297872
+685,1,20130417,1,10:06:29,95,0.631578947
+686,1,20130417,1,10:06:29,102,0.588235294
+687,1,20130417,1,10:06:30,99,0.606060606
+688,1,20130417,1,10:06:30,99,0.606060606
+689,1,20130417,1,10:06:31,104,0.576923077
+690,1,20130417,1,10:06:32,102,0.588235294
+691,1,20130417,1,10:06:32,106,0.566037736
+692,1,20130417,1,10:06:33,107,0.560747664
+693,1,20130417,1,10:06:33,107,0.560747664
+694,1,20130417,1,10:06:34,107,0.560747664
+695,1,20130417,1,10:06:34,109,0.550458716
+696,1,20130417,1,10:06:35,109,0.550458716
+697,1,20130417,1,10:06:36,107,0.560747664
+698,1,20130417,1,10:06:36,112,0.535714286
+699,1,20130417,1,10:06:37,109,0.550458716
+700,1,20130417,1,10:06:37,109,0.550458716
+701,1,20130417,1,10:06:38,112,0.535714286
+702,1,20130417,1,10:06:38,111,0.540540541
+703,1,20130417,1,10:06:39,110,0.545454545
+704,1,20130417,1,10:06:39,111,0.540540541
+705,1,20130417,1,10:06:40,108,0.555555556
+706,1,20130417,1,10:06:40,106,0.566037736
+707,1,20130417,1,10:06:41,108,0.555555556
+708,1,20130417,1,10:06:42,107,0.560747664
+709,1,20130417,1,10:06:42,102,0.588235294
+710,1,20130417,1,10:06:43,102,0.588235294
+711,1,20130417,1,10:06:43,100,0.6
+712,1,20130417,1,10:06:44,107,0.560747664
+713,1,20130417,1,10:06:45,96,0.625
+714,1,20130417,1,10:06:45,101,0.594059406
+715,1,20130417,1,10:06:46,105,0.571428571
+716,1,20130417,1,10:06:46,96,0.625
+717,1,20130417,1,10:06:47,103,0.582524272
+718,1,20130417,1,10:06:48,105,0.571428571
+719,1,20130417,1,10:06:48,105,0.571428571
+720,1,20130417,1,10:06:49,104,0.576923077
+721,1,20130417,1,10:06:49,106,0.566037736
+722,1,20130417,1,10:06:50,106,0.566037736
+723,1,20130417,1,10:06:50,105,0.571428571
+724,1,20130417,1,10:06:51,105,0.571428571
+725,1,20130417,1,10:06:52,108,0.555555556
+726,1,20130417,1,10:06:52,105,0.571428571
+727,1,20130417,1,10:06:53,103,0.582524272
+728,1,20130417,1,10:06:53,98,0.612244898
+729,1,20130417,1,10:06:54,103,0.582524272
+730,1,20130417,1,10:06:54,105,0.571428571
+731,1,20130417,1,10:06:55,105,0.571428571
+732,1,20130417,1,10:06:56,105,0.571428571
+733,1,20130417,1,10:06:56,107,0.560747664
+734,1,20130417,1,10:06:57,107,0.560747664
+735,1,20130417,1,10:06:57,92,0.652173913
+736,1,20130417,1,10:06:58,87,0.689655172
+737,1,20130417,1,10:06:59,83,0.722891566
+738,1,20130417,1,10:06:59,70,0.857142857
+739,1,20130417,1,10:07:00,70,0.857142857
+740,1,20130417,1,10:07:01,73,0.821917808
+741,1,20130417,1,10:07:02,77,0.779220779
+742,1,20130417,1,10:07:03,82,0.731707317
+743,1,20130417,1,10:07:03,89,0.674157303
+744,1,20130417,1,10:07:04,87,0.689655172
+745,1,20130417,1,10:07:05,82,0.731707317
+746,1,20130417,1,10:07:05,88,0.681818182
+747,1,20130417,1,10:07:06,85,0.705882353
+748,1,20130417,1,10:07:07,74,0.810810811
+749,1,20130417,1,10:07:08,72,0.833333333
+750,1,20130417,1,10:07:08,73,0.821917808
+751,1,20130417,1,10:07:09,68,0.882352941
+752,1,20130417,1,10:07:10,70,0.857142857
+753,1,20130417,1,10:07:11,81,0.740740741
+754,1,20130417,1,10:07:12,81,0.740740741
+755,1,20130417,1,10:07:13,77,0.779220779
+756,1,20130417,1,10:07:13,78,0.769230769
+757,1,20130417,1,10:07:14,85,0.705882353
+758,1,20130417,1,10:07:15,83,0.722891566
+759,1,20130417,1,10:07:16,72,0.833333333
+760,1,20130417,1,10:07:16,69,0.869565217
+761,1,20130417,1,10:07:17,76,0.789473684
+762,1,20130417,1,10:07:18,75,0.8
+763,1,20130417,1,10:07:19,69,0.869565217
+764,1,20130417,1,10:07:20,71,0.845070423
+765,1,20130417,1,10:07:21,82,0.731707317
+766,1,20130417,1,10:07:21,79,0.759493671
+767,1,20130417,1,10:07:22,74,0.810810811
+768,1,20130417,1,10:07:23,74,0.810810811
+769,1,20130417,1,10:07:24,78,0.769230769
+770,1,20130417,1,10:07:24,65,0.923076923
+771,1,20130417,1,10:07:25,64,0.9375
+772,1,20130417,1,10:07:26,75,0.8
+773,1,20130417,1,10:07:27,70,0.857142857
+774,1,20130417,1,10:07:28,67,0.895522388
+775,1,20130417,1,10:07:29,74,0.810810811
+776,1,20130417,1,10:07:30,74,0.810810811
+777,1,20130417,1,10:07:30,72,0.833333333
+778,1,20130417,1,10:07:31,80,0.75
+779,1,20130417,1,10:07:32,81,0.740740741
+780,1,20130417,1,10:07:33,73,0.821917808
+781,1,20130417,1,10:07:34,69,0.869565217
+782,1,20130417,1,10:07:34,79,0.759493671
+783,1,20130417,1,10:07:35,77,0.779220779
+784,1,20130417,1,10:07:36,72,0.833333333
+785,1,20130417,1,10:07:37,79,0.759493671
+786,1,20130417,1,10:07:38,87,0.689655172
+787,1,20130417,1,10:07:38,87,0.689655172
+788,1,20130417,1,10:07:39,88,0.681818182
+789,1,20130417,1,10:07:40,87,0.689655172
+790,1,20130417,1,10:07:40,79,0.759493671
+791,1,20130417,1,10:07:41,75,0.8
+792,1,20130417,1,10:07:42,68,0.882352941
+793,1,20130417,1,10:07:43,72,0.833333333
+794,1,20130417,1,10:07:44,64,0.9375
+795,1,20130417,1,10:07:45,69,0.869565217
+796,1,20130417,1,10:07:45,76,0.789473684
+797,1,20130417,1,10:07:46,73,0.821917808
+798,1,20130417,1,10:07:47,65,0.923076923
+799,1,20130417,1,10:07:48,75,0.8
+800,1,20130417,1,10:07:49,74,0.810810811
+801,1,20130417,1,10:07:50,71,0.845070423
+802,1,20130417,1,10:07:50,78,0.769230769
+803,1,20130417,1,10:07:51,82,0.731707317
+804,1,20130417,1,10:07:52,68,0.882352941
+805,1,20130417,1,10:07:53,71,0.845070423
+806,1,20130417,1,10:07:54,72,0.833333333
+807,1,20130417,1,10:07:54,67,0.895522388
+808,1,20130417,1,10:07:55,66,0.909090909
+809,1,20130417,1,10:07:56,68,0.882352941
+810,1,20130417,1,10:07:57,70,0.857142857
+811,1,20130417,1,10:07:58,68,0.882352941
+812,1,20130417,1,10:07:59,73,0.821917808
+813,1,20130417,1,10:08:00,80,0.75
+814,1,20130417,1,10:08:00,72,0.833333333
+815,1,20130417,1,10:08:01,70,0.857142857
+816,1,20130417,1,10:08:02,74,0.810810811
+817,1,20130417,1,10:08:03,81,0.740740741
+818,1,20130417,1,10:08:04,83,0.722891566
+819,1,20130417,1,10:08:04,75,0.8
+820,1,20130417,1,10:08:05,65,0.923076923
+821,1,20130417,1,10:08:06,67,0.895522388
+822,1,20130417,1,10:08:07,65,0.923076923
+823,1,20130417,1,10:08:08,65,0.923076923
+824,1,20130417,1,10:08:09,68,0.882352941
+825,1,20130417,1,10:08:10,70,0.857142857
+826,1,20130417,1,10:08:11,67,0.895522388
+827,1,20130417,1,10:08:12,67,0.895522388
+828,1,20130417,1,10:08:12,75,0.8
+829,1,20130417,1,10:08:13,76,0.789473684
+830,1,20130417,1,10:08:14,73,0.821917808
+831,1,20130417,1,10:08:15,74,0.810810811
+832,1,20130417,1,10:08:16,74,0.810810811
+833,1,20130417,1,10:08:16,70,0.857142857
+834,1,20130417,1,10:08:17,71,0.845070423
+835,1,20130417,1,10:08:18,75,0.8
+836,1,20130417,1,10:08:19,70,0.857142857
+837,1,20130417,1,10:08:20,67,0.895522388
+838,1,20130417,1,10:08:21,77,0.779220779
+839,1,20130417,1,10:08:22,80,0.75
+840,1,20130417,1,10:08:22,84,0.714285714
+841,1,20130417,1,10:08:23,88,0.681818182
+842,1,20130417,1,10:08:24,92,0.652173913
+843,1,20130417,1,10:08:24,90,0.666666667
+844,1,20130417,1,10:08:25,93,0.64516129
+845,1,20130417,1,10:08:26,78,0.769230769
+846,1,20130417,1,10:08:26,78,0.769230769
+847,1,20130417,1,10:08:27,79,0.759493671
+848,1,20130417,1,10:08:28,73,0.821917808
+849,1,20130417,1,10:08:29,68,0.882352941
+850,1,20130417,1,10:08:30,76,0.789473684
+851,1,20130417,1,10:08:30,74,0.810810811
+852,1,20130417,1,10:08:31,71,0.845070423
+853,1,20130417,1,10:08:32,75,0.8
+854,1,20130417,1,10:08:33,76,0.789473684
+855,1,20130417,1,10:08:34,72,0.833333333
+856,1,20130417,1,10:08:34,67,0.895522388
+857,1,20130417,1,10:08:35,74,0.810810811
+858,1,20130417,1,10:08:36,69,0.869565217
+859,1,20130417,1,10:08:37,67,0.895522388
+860,1,20130417,1,10:08:38,68,0.882352941
+861,1,20130417,1,10:08:39,72,0.833333333
+862,1,20130417,1,10:08:40,67,0.895522388
+863,1,20130417,1,10:08:41,66,0.909090909
+864,1,20130417,1,10:08:41,72,0.833333333
+865,1,20130417,1,10:08:42,71,0.845070423
+866,1,20130417,1,10:08:43,68,0.882352941
+867,1,20130417,1,10:08:44,72,0.833333333
+868,1,20130417,1,10:08:45,77,0.779220779
+869,1,20130417,1,10:08:46,74,0.810810811
+870,1,20130417,1,10:08:46,75,0.8
+871,1,20130417,1,10:08:47,77,0.779220779
+872,1,20130417,1,10:08:48,64,0.9375
+873,1,20130417,1,10:08:49,62,0.967741935
+874,1,20130417,1,10:08:50,70,0.857142857
+875,1,20130417,1,10:08:51,68,0.882352941
+876,1,20130417,1,10:08:52,75,0.8
+877,1,20130417,1,10:08:52,71,0.845070423
+878,1,20130417,1,10:08:53,67,0.895522388
+879,1,20130417,1,10:08:54,66,0.909090909
+880,1,20130417,1,10:08:55,74,0.810810811
+881,1,20130417,1,10:08:56,67,0.895522388
+882,1,20130417,1,10:08:57,68,0.882352941
+883,1,20130417,1,10:08:58,72,0.833333333
+884,1,20130417,1,10:08:59,65,0.923076923
+885,1,20130417,1,10:08:59,61,0.983606557
+886,1,20130417,1,10:09:00,68,0.882352941
+887,1,20130417,1,10:09:01,70,0.857142857
+888,1,20130417,1,10:09:02,67,0.895522388
+889,1,20130417,1,10:09:03,67,0.895522388
+890,1,20130417,1,10:09:04,74,0.810810811
+891,1,20130417,1,10:09:05,70,0.857142857
+892,1,20130417,1,10:09:06,67,0.895522388
+893,1,20130417,1,10:09:07,73,0.821917808
+894,1,20130417,1,10:09:07,77,0.779220779
+895,1,20130417,1,10:09:08,74,0.810810811
+896,1,20130417,1,10:09:09,77,0.779220779
+897,1,20130417,1,10:09:10,68,0.882352941
+898,1,20130417,1,10:09:11,61,0.983606557
+899,1,20130417,1,10:09:12,69,0.869565217
+900,1,20130417,1,10:09:12,72,0.833333333
+901,1,20130417,1,10:09:13,67,0.895522388
+902,1,20130417,1,10:09:14,66,0.909090909
+903,1,20130417,1,10:09:15,70,0.857142857
+904,1,20130417,1,10:09:16,72,0.833333333
+905,1,20130417,1,10:09:17,68,0.882352941
+906,1,20130417,1,10:09:18,68,0.882352941
+907,1,20130417,1,10:09:19,73,0.821917808
+908,1,20130417,1,10:09:19,71,0.845070423
+909,1,20130417,1,10:09:20,64,0.9375
+910,1,20130417,1,10:09:21,70,0.857142857
+911,1,20130417,1,10:09:22,68,0.882352941
+912,1,20130417,1,10:09:23,63,0.952380952
+913,1,20130417,1,10:09:24,64,0.9375
+914,1,20130417,1,10:09:25,76,0.789473684
+915,1,20130417,1,10:09:26,76,0.789473684
+916,1,20130417,1,10:09:26,75,0.8
+917,1,20130417,1,10:09:27,79,0.759493671
+918,1,20130417,1,10:09:28,80,0.75
+919,1,20130417,1,10:09:29,68,0.882352941
+920,1,20130417,1,10:09:30,66,0.909090909
+921,1,20130417,1,10:09:30,69,0.869565217
+922,1,20130417,1,10:09:31,66,0.909090909
+923,1,20130417,1,10:09:32,62,0.967741935
+924,1,20130417,1,10:09:33,67,0.895522388
+925,1,20130417,1,10:09:34,73,0.821917808
+926,1,20130417,1,10:09:35,71,0.845070423
+927,1,20130417,1,10:09:36,67,0.895522388
+928,1,20130417,1,10:09:37,74,0.810810811
+929,1,20130417,1,10:09:37,68,0.882352941
+930,1,20130417,1,10:09:38,61,0.983606557
+931,1,20130417,1,10:09:39,74,0.810810811
+932,1,20130417,1,10:09:40,69,0.869565217
+933,1,20130417,1,10:09:41,64,0.9375
+934,1,20130417,1,10:09:42,66,0.909090909
+935,1,20130417,1,10:09:43,74,0.810810811
+936,1,20130417,1,10:09:44,64,0.9375
+937,1,20130417,1,10:09:45,63,0.952380952
+938,1,20130417,1,10:09:46,68,0.882352941
+939,1,20130417,1,10:09:46,76,0.789473684
+940,1,20130417,1,10:09:47,69,0.869565217
+941,1,20130417,1,10:09:48,59,1.016949153
+942,1,20130417,1,10:09:49,67,0.895522388
+943,1,20130417,1,10:09:50,73,0.821917808
+944,1,20130417,1,10:09:51,69,0.869565217
+945,1,20130417,1,10:09:52,65,0.923076923
+946,1,20130417,1,10:09:53,65,0.923076923
+947,1,20130417,1,10:09:54,71,0.845070423
+948,1,20130417,1,10:09:54,64,0.9375
+949,1,20130417,1,10:09:55,63,0.952380952
+950,1,20130417,1,10:09:56,68,0.882352941
+951,1,20130417,1,10:09:57,69,0.869565217
+952,1,20130417,1,10:09:58,65,0.923076923
+953,1,20130417,1,10:09:59,64,0.9375
+954,1,20130417,1,10:10:00,72,0.833333333
+955,1,20130417,1,10:10:01,68,0.882352941
+956,1,20130417,1,10:10:02,67,0.895522388
+957,1,20130417,1,10:10:03,70,0.857142857
+958,1,20130417,1,10:10:03,69,0.869565217
+959,1,20130417,1,10:10:04,64,0.9375
+960,1,20130417,1,10:10:05,65,0.923076923
+961,1,20130417,1,10:10:06,72,0.833333333
+962,1,20130417,1,10:10:07,69,0.869565217
+963,1,20130417,1,10:10:08,65,0.923076923
+964,1,20130417,1,10:10:09,67,0.895522388
+965,1,20130417,1,10:10:10,71,0.845070423
+966,1,20130417,1,10:10:10,65,0.923076923
+967,1,20130417,1,10:10:11,66,0.909090909
+968,1,20130417,1,10:10:12,68,0.882352941
+969,1,20130417,1,10:10:13,68,0.882352941
+970,1,20130417,1,10:10:14,66,0.909090909
+971,1,20130417,1,10:10:15,70,0.857142857
+972,1,20130417,1,10:10:16,73,0.821917808
+973,1,20130417,1,10:10:17,69,0.869565217
+974,1,20130417,1,10:10:18,68,0.882352941
+975,1,20130417,1,10:10:18,75,0.8
+976,1,20130417,1,10:10:19,73,0.821917808
+977,1,20130417,1,10:10:20,72,0.833333333
+978,1,20130417,1,10:10:21,75,0.8
+979,1,20130417,1,10:10:22,78,0.769230769
+980,1,20130417,1,10:10:22,69,0.869565217
+981,1,20130417,1,10:10:23,65,0.923076923
+982,1,20130417,1,10:10:24,71,0.845070423
+983,1,20130417,1,10:10:25,69,0.869565217
+984,1,20130417,1,10:10:26,69,0.869565217
+985,1,20130417,1,10:10:27,77,0.779220779
+986,1,20130417,1,10:10:28,79,0.759493671
+987,1,20130417,1,10:10:28,75,0.8
+988,1,20130417,1,10:10:29,72,0.833333333
+989,1,20130417,1,10:10:30,76,0.789473684
+990,1,20130417,1,10:10:31,71,0.845070423
+991,1,20130417,1,10:10:32,65,0.923076923
+992,1,20130417,1,10:10:33,71,0.845070423
+993,1,20130417,1,10:10:33,77,0.779220779
+994,1,20130417,1,10:10:34,73,0.821917808
+995,1,20130417,1,10:10:35,78,0.769230769
+996,1,20130417,1,10:10:36,80,0.75
+997,1,20130417,1,10:10:37,83,0.722891566
+998,1,20130417,1,10:10:37,82,0.731707317
+999,1,20130417,1,10:10:38,83,0.722891566
+1000,1,20130417,1,10:10:39,79,0.759493671
+1001,1,20130417,1,10:10:39,81,0.740740741
+1002,1,20130417,1,10:10:40,82,0.731707317
+1003,1,20130417,1,10:10:41,83,0.722891566
+1004,1,20130417,1,10:10:42,80,0.75
+1005,1,20130417,1,10:10:42,67,0.895522388
+1006,1,20130417,1,10:10:43,72,0.833333333
+1007,1,20130417,1,10:10:44,77,0.779220779
+1008,1,20130417,1,10:10:45,76,0.789473684
+1009,1,20130417,1,10:10:46,74,0.810810811
+1010,1,20130417,1,10:10:47,73,0.821917808
+1011,1,20130417,1,10:10:47,77,0.779220779
+1012,1,20130417,1,10:10:48,74,0.810810811
+1013,1,20130417,1,10:10:49,70,0.857142857
+1014,1,20130417,1,10:10:50,67,0.895522388
+1015,1,20130417,1,10:10:51,74,0.810810811
+1016,1,20130417,1,10:10:52,72,0.833333333
+1017,1,20130417,1,10:10:52,68,0.882352941
+1018,1,20130417,1,10:10:53,67,0.895522388
+1019,1,20130417,1,10:10:54,70,0.857142857
+1020,1,20130417,1,10:10:55,67,0.895522388
+1021,1,20130417,1,10:10:56,66,0.909090909
+1022,1,20130417,1,10:10:57,67,0.895522388
+1023,1,20130417,1,10:10:58,74,0.810810811
+1024,1,20130417,1,10:10:59,72,0.833333333
+1025,1,20130417,1,10:10:59,69,0.869565217
+1026,1,20130417,1,10:11:00,71,0.845070423
+1027,1,20130417,1,10:11:01,74,0.810810811
+1028,1,20130417,1,10:11:02,72,0.833333333
+1029,1,20130417,1,10:11:03,75,0.8
+1030,1,20130417,1,10:11:03,78,0.769230769
+1031,1,20130417,1,10:11:04,85,0.705882353
+1032,1,20130417,1,10:11:05,82,0.731707317
+1033,1,20130417,1,10:11:06,80,0.75
+1034,1,20130417,1,10:11:06,78,0.769230769
+1035,1,20130417,1,10:11:07,79,0.759493671
+1036,1,20130417,1,10:11:08,79,0.759493671
+1037,1,20130417,1,10:11:09,81,0.740740741
+1038,1,20130417,1,10:11:09,82,0.731707317
+1039,1,20130417,1,10:11:10,82,0.731707317
+1040,1,20130417,1,10:11:11,82,0.731707317
+1041,1,20130417,1,10:11:12,83,0.722891566
+1042,1,20130417,1,10:11:12,84,0.714285714
+1043,1,20130417,1,10:11:13,78,0.769230769
+1044,1,20130417,1,10:11:14,72,0.833333333
+1045,1,20130417,1,10:11:15,71,0.845070423
+1046,1,20130417,1,10:11:16,71,0.845070423
+1047,1,20130417,1,10:11:16,70,0.857142857
+1048,1,20130417,1,10:11:17,70,0.857142857
+1049,1,20130417,1,10:11:18,71,0.845070423
+1050,1,20130417,1,10:11:19,76,0.789473684
+1051,1,20130417,1,10:11:20,78,0.769230769
+1052,1,20130417,1,10:11:21,78,0.769230769
+1053,1,20130417,1,10:11:21,74,0.810810811
+1054,1,20130417,1,10:11:22,72,0.833333333
+1055,1,20130417,1,10:11:23,70,0.857142857
+1056,1,20130417,1,10:11:24,75,0.8
+1057,1,20130417,1,10:11:25,78,0.769230769
+1058,1,20130417,1,10:11:25,81,0.740740741
+1059,1,20130417,1,10:11:26,82,0.731707317
+1060,1,20130417,1,10:11:27,83,0.722891566
+1061,1,20130417,1,10:11:28,87,0.689655172
+1062,1,20130417,1,10:11:28,86,0.697674419
+1063,1,20130417,1,10:11:29,83,0.722891566
+1064,1,20130417,1,10:11:30,84,0.714285714
+1065,1,20130417,1,10:11:30,83,0.722891566
+1066,1,20130417,1,10:11:31,78,0.769230769
+1067,1,20130417,1,10:11:32,74,0.810810811
+1068,1,20130417,1,10:11:33,75,0.8
+1069,1,20130417,1,10:11:33,74,0.810810811
+1070,1,20130417,1,10:11:34,80,0.75
+1071,1,20130417,1,10:11:35,81,0.740740741
+1072,1,20130417,1,10:11:36,81,0.740740741
+1073,1,20130417,1,10:11:37,78,0.769230769
+1074,1,20130417,1,10:11:37,73,0.821917808
+1075,1,20130417,1,10:11:38,78,0.769230769
+1076,1,20130417,1,10:11:39,76,0.789473684
+1077,1,20130417,1,10:11:40,73,0.821917808
+1078,1,20130417,1,10:11:40,71,0.845070423
+1079,1,20130417,1,10:11:41,72,0.833333333
+1080,1,20130417,1,10:11:42,76,0.789473684
+1081,1,20130417,1,10:11:43,75,0.8
+1082,1,20130417,1,10:11:44,78,0.769230769
+1083,1,20130417,1,10:11:45,78,0.769230769
+1084,1,20130417,1,10:11:45,84,0.714285714
+1085,1,20130417,1,10:11:46,84,0.714285714
+1086,1,20130417,1,10:11:47,82,0.731707317
+1087,1,20130417,1,10:11:47,76,0.789473684
+1088,1,20130417,1,10:11:48,76,0.789473684
+1089,1,20130417,1,10:11:49,80,0.75
+1090,1,20130417,1,10:11:50,82,0.731707317
+1091,1,20130417,1,10:11:51,77,0.779220779
+1092,1,20130417,1,10:11:51,70,0.857142857
+1093,1,20130417,1,10:11:52,80,0.75
+1094,1,20130417,1,10:11:53,81,0.740740741
+1095,1,20130417,1,10:11:54,81,0.740740741
+1096,1,20130417,1,10:11:54,84,0.714285714
+1097,1,20130417,1,10:11:55,86,0.697674419
+1098,1,20130417,1,10:11:56,89,0.674157303
+1099,1,20130417,1,10:11:56,88,0.681818182
+1100,1,20130417,1,10:11:57,87,0.689655172
+1101,1,20130417,1,10:11:58,85,0.705882353
+1102,1,20130417,1,10:11:59,88,0.681818182
+1103,1,20130417,1,10:11:59,90,0.666666667
+1104,1,20130417,1,10:12:00,90,0.666666667
+1105,1,20130417,1,10:12:01,88,0.681818182
+1106,1,20130417,1,10:12:01,85,0.705882353
+1107,1,20130417,1,10:12:02,84,0.714285714
+1108,1,20130417,1,10:12:03,86,0.697674419
+1109,1,20130417,1,10:12:03,87,0.689655172
+1110,1,20130417,1,10:12:04,84,0.714285714
+1111,1,20130417,1,10:12:05,81,0.740740741
+1112,1,20130417,1,10:12:06,77,0.779220779
+1113,1,20130417,1,10:12:06,76,0.789473684
+1114,1,20130417,1,10:12:07,81,0.740740741
+1115,1,20130417,1,10:12:08,82,0.731707317
+1116,1,20130417,1,10:12:09,78,0.769230769
+1117,1,20130417,1,10:12:09,75,0.8
+1118,1,20130417,1,10:12:10,73,0.821917808
+1119,1,20130417,1,10:12:11,76,0.789473684
+1120,1,20130417,1,10:12:12,76,0.789473684
+1121,1,20130417,1,10:12:13,72,0.833333333
+1122,1,20130417,1,10:12:13,70,0.857142857
+1123,1,20130417,1,10:12:14,75,0.8
+1124,1,20130417,1,10:12:15,83,0.722891566
+1125,1,20130417,1,10:12:16,85,0.705882353
+1126,1,20130417,1,10:12:16,86,0.697674419
+1127,1,20130417,1,10:12:17,89,0.674157303
+1128,1,20130417,1,10:12:18,84,0.714285714
+1129,1,20130417,1,10:12:19,77,0.779220779
+1130,1,20130417,1,10:12:19,79,0.759493671
+1131,1,20130417,1,10:12:20,78,0.769230769
+1132,1,20130417,1,10:12:21,71,0.845070423
+1133,1,20130417,1,10:12:22,67,0.895522388
+1134,1,20130417,1,10:12:23,74,0.810810811
+1135,1,20130417,1,10:12:23,78,0.769230769
+1136,1,20130417,1,10:12:24,76,0.789473684
+1137,1,20130417,1,10:12:25,72,0.833333333
+1138,1,20130417,1,10:12:26,71,0.845070423
+1139,1,20130417,1,10:12:27,76,0.789473684
+1140,1,20130417,1,10:12:27,70,0.857142857
+1141,1,20130417,1,10:12:28,61,0.983606557
+1142,1,20130417,1,10:12:29,66,0.909090909
+1143,1,20130417,1,10:12:30,72,0.833333333
+1144,1,20130417,1,10:12:31,65,0.923076923
+1145,1,20130417,1,10:12:32,62,0.967741935
+1146,1,20130417,1,10:12:33,71,0.845070423
+1147,1,20130417,1,10:12:34,77,0.779220779
+1148,1,20130417,1,10:12:35,75,0.8
+1149,1,20130417,1,10:12:35,74,0.810810811
+1150,1,20130417,1,10:12:36,71,0.845070423
+1151,1,20130417,1,10:12:37,77,0.779220779
+1152,1,20130417,1,10:12:38,73,0.821917808
+1153,1,20130417,1,10:12:39,71,0.845070423
+1154,1,20130417,1,10:12:39,67,0.895522388
+1155,1,20130417,1,10:12:40,73,0.821917808
+1156,1,20130417,1,10:12:41,69,0.869565217
+1157,1,20130417,1,10:12:42,66,0.909090909
+1158,1,20130417,1,10:12:43,71,0.845070423
+1159,1,20130417,1,10:12:44,77,0.779220779
+1160,1,20130417,1,10:12:45,77,0.779220779
+1161,1,20130417,1,10:12:45,72,0.833333333
+1162,1,20130417,1,10:12:46,71,0.845070423
+1163,1,20130417,1,10:12:47,72,0.833333333
+1164,1,20130417,1,10:12:48,69,0.869565217
+1165,1,20130417,1,10:12:49,63,0.952380952
+1166,1,20130417,1,10:12:50,67,0.895522388
+1167,1,20130417,1,10:12:51,71,0.845070423
+1168,1,20130417,1,10:12:51,66,0.909090909
+1169,1,20130417,1,10:12:52,67,0.895522388
+1170,1,20130417,1,10:12:53,72,0.833333333
+1171,1,20130417,1,10:12:54,73,0.821917808
+1172,1,20130417,1,10:12:55,69,0.869565217
+1173,1,20130417,1,10:12:56,63,0.952380952
+1174,1,20130417,1,10:12:57,71,0.845070423
+1175,1,20130417,1,10:12:58,79,0.759493671
+1176,1,20130417,1,10:12:58,74,0.810810811
+1177,1,20130417,1,10:12:59,62,0.967741935
+1178,1,20130417,1,10:13:00,62,0.967741935
+1179,1,20130417,1,10:13:01,70,0.857142857
+1180,1,20130417,1,10:13:02,66,0.909090909
+1181,1,20130417,1,10:13:03,64,0.9375
+1182,1,20130417,1,10:13:04,62,0.967741935
+1183,1,20130417,1,10:13:05,69,0.869565217
+1184,1,20130417,1,10:13:06,67,0.895522388
+1185,1,20130417,1,10:13:07,60,1
+1186,1,20130417,1,10:13:08,64,0.9375
+1187,1,20130417,1,10:13:08,71,0.845070423
+1188,1,20130417,1,10:13:09,71,0.845070423
+1189,1,20130417,1,10:13:10,65,0.923076923
+1190,1,20130417,1,10:13:11,67,0.895522388
+1191,1,20130417,1,10:13:12,57,1.052631579
+1192,1,20130417,1,10:13:13,61,0.983606557
+1193,1,20130417,1,10:13:14,67,0.895522388
+1194,1,20130417,1,10:13:15,71,0.845070423
+1195,1,20130417,1,10:13:16,64,0.9375
+1196,1,20130417,1,10:13:17,61,0.983606557
+1197,1,20130417,1,10:13:18,65,0.923076923
+1198,1,20130417,1,10:13:19,73,0.821917808
+1199,1,20130417,1,10:13:19,72,0.833333333
+1200,1,20130417,1,10:13:20,72,0.833333333
+1201,1,20130417,1,10:13:21,70,0.857142857
+1202,1,20130417,1,10:13:22,72,0.833333333
+1203,1,20130417,1,10:13:23,65,0.923076923
+1204,1,20130417,1,10:13:24,64,0.9375
+1205,1,20130417,1,10:13:25,66,0.909090909
+1206,1,20130417,1,10:13:26,72,0.833333333
+1207,1,20130417,1,10:13:26,71,0.845070423
+1208,1,20130417,1,10:13:27,69,0.869565217
+1209,1,20130417,1,10:13:28,69,0.869565217
+1210,1,20130417,1,10:13:29,76,0.789473684
+1211,1,20130417,1,10:13:30,72,0.833333333
+1212,1,20130417,1,10:13:31,65,0.923076923
+1213,1,20130417,1,10:13:31,60,1
+1214,1,20130417,1,10:13:32,67,0.895522388
+1215,1,20130417,1,10:13:33,73,0.821917808
+1216,1,20130417,1,10:13:34,69,0.869565217
+1217,1,20130417,1,10:13:35,66,0.909090909
+1218,1,20130417,1,10:13:36,73,0.821917808
+1219,1,20130417,1,10:13:37,73,0.821917808
+1220,1,20130417,1,10:13:38,68,0.882352941
+1221,1,20130417,1,10:13:39,69,0.869565217
+1222,1,20130417,1,10:13:39,71,0.845070423
+1223,1,20130417,1,10:13:40,66,0.909090909
+1224,1,20130417,1,10:13:41,77,0.779220779
+1225,1,20130417,1,10:13:42,81,0.740740741
+1226,1,20130417,1,10:13:43,74,0.810810811
+1227,1,20130417,1,10:13:43,70,0.857142857
+1228,1,20130417,1,10:13:44,61,0.983606557
+1229,1,20130417,1,10:13:45,63,0.952380952
+1230,1,20130417,1,10:13:46,71,0.845070423
+1231,1,20130417,1,10:13:47,76,0.789473684
+1232,1,20130417,1,10:13:48,81,0.740740741
+1233,1,20130417,1,10:13:49,84,0.714285714
+1234,1,20130417,1,10:13:49,87,0.689655172
+1235,1,20130417,1,10:13:50,83,0.722891566
+1236,1,20130417,1,10:13:51,83,0.722891566
+1237,1,20130417,1,10:13:52,76,0.789473684
+1238,1,20130417,1,10:13:52,65,0.923076923
+1239,1,20130417,1,10:13:53,61,0.983606557
+1240,1,20130417,1,10:13:54,61,0.983606557
+1241,1,20130417,1,10:13:55,63,0.952380952
+1242,1,20130417,1,10:13:56,61,0.983606557
+1243,1,20130417,1,10:13:57,63,0.952380952
+1244,1,20130417,1,10:13:58,62,0.967741935
+1245,1,20130417,1,10:13:59,65,0.923076923
+1246,1,20130417,1,10:14:00,64,0.9375
+1247,1,20130417,1,10:14:01,63,0.952380952
+1248,1,20130417,1,10:14:02,74,0.810810811
+1249,1,20130417,1,10:14:03,68,0.882352941
+1250,1,20130417,1,10:14:04,67,0.895522388
+1251,1,20130417,1,10:14:04,69,0.869565217
+1252,1,20130417,1,10:14:05,61,0.983606557
+1253,1,20130417,1,10:14:06,72,0.833333333
+1254,1,20130417,1,10:14:07,68,0.882352941
+1255,1,20130417,1,10:14:08,56,1.071428571
+1256,1,20130417,1,10:14:09,59,1.016949153
+1257,1,20130417,1,10:14:10,61,0.983606557
+1258,1,20130417,1,10:14:11,66,0.909090909
+1259,1,20130417,1,10:14:12,67,0.895522388
+1260,1,20130417,1,10:14:13,65,0.923076923
+1261,1,20130417,1,10:14:14,67,0.895522388
+1262,1,20130417,1,10:14:15,69,0.869565217
+1263,1,20130417,1,10:14:16,78,0.769230769
+1264,1,20130417,1,10:14:16,76,0.789473684
+1265,1,20130417,1,10:14:17,69,0.869565217
+1266,1,20130417,1,10:14:18,65,0.923076923
+1267,1,20130417,1,10:14:19,67,0.895522388
+1268,1,20130417,1,10:14:20,70,0.857142857
+1269,1,20130417,1,10:14:21,64,0.9375
+1270,1,20130417,1,10:14:22,60,1
+1271,1,20130417,1,10:14:23,67,0.895522388
+1272,1,20130417,1,10:14:24,71,0.845070423
+1273,1,20130417,1,10:14:24,73,0.821917808
+1274,1,20130417,1,10:14:25,77,0.779220779
+1275,1,20130417,1,10:14:26,83,0.722891566
+1276,1,20130417,1,10:14:27,84,0.714285714
+1277,1,20130417,1,10:14:27,81,0.740740741
+1278,1,20130417,1,10:14:28,81,0.740740741
+1279,1,20130417,1,10:14:29,83,0.722891566
+1280,1,20130417,1,10:14:30,85,0.705882353
+1281,1,20130417,1,10:14:30,83,0.722891566
+1282,1,20130417,1,10:14:31,80,0.75
+1283,1,20130417,1,10:14:32,77,0.779220779
+1284,1,20130417,1,10:14:33,82,0.731707317
+1285,1,20130417,1,10:14:33,82,0.731707317
+1286,1,20130417,1,10:14:34,80,0.75
+1287,1,20130417,1,10:14:35,68,0.882352941
+1288,1,20130417,1,10:14:36,77,0.779220779
+1289,1,20130417,1,10:14:36,75,0.8
+1290,1,20130417,1,10:14:37,68,0.882352941
+1291,1,20130417,1,10:14:38,66,0.909090909
+1292,1,20130417,1,10:14:39,72,0.833333333
+1293,1,20130417,1,10:14:40,67,0.895522388
+1294,1,20130417,1,10:14:41,63,0.952380952
+1295,1,20130417,1,10:14:42,69,0.869565217
+1296,1,20130417,1,10:14:43,69,0.869565217
+1297,1,20130417,1,10:14:43,67,0.895522388
+1298,1,20130417,1,10:14:44,67,0.895522388
+1299,1,20130417,1,10:14:45,71,0.845070423
+1300,1,20130417,1,10:14:46,64,0.9375
+1301,1,20130417,1,10:14:47,59,1.016949153
+1302,1,20130417,1,10:14:48,66,0.909090909
+1303,1,20130417,1,10:14:49,62,0.967741935
+1304,1,20130417,1,10:14:50,61,0.983606557
+1305,1,20130417,1,10:14:51,69,0.869565217
+1306,1,20130417,1,10:14:52,65,0.923076923
+1307,1,20130417,1,10:14:53,62,0.967741935
+1308,1,20130417,1,10:14:54,67,0.895522388
+1309,1,20130417,1,10:14:55,65,0.923076923
+1310,1,20130417,1,10:14:55,60,1
+1311,1,20130417,1,10:14:56,60,1
+1312,1,20130417,1,10:14:57,67,0.895522388
+1313,1,20130417,1,10:14:58,61,0.983606557
+1314,1,20130417,1,10:14:59,61,0.983606557
+1315,1,20130417,1,10:15:00,67,0.895522388
+1316,1,20130417,1,10:15:01,67,0.895522388
+1317,1,20130417,1,10:15:02,61,0.983606557
+1318,1,20130417,1,10:15:03,62,0.967741935
+1319,1,20130417,1,10:15:04,67,0.895522388
+1320,1,20130417,1,10:15:05,63,0.952380952
+1321,1,20130417,1,10:15:06,60,1
+1322,1,20130417,1,10:15:07,67,0.895522388
+1323,1,20130417,1,10:15:08,68,0.882352941
+1324,1,20130417,1,10:15:09,67,0.895522388
+1325,1,20130417,1,10:15:10,70,0.857142857
+1326,1,20130417,1,10:15:10,73,0.821917808
+1327,1,20130417,1,10:15:11,63,0.952380952
+1328,1,20130417,1,10:15:12,57,1.052631579
+1329,1,20130417,1,10:15:13,67,0.895522388
+1330,1,20130417,1,10:15:14,62,0.967741935
+1331,1,20130417,1,10:15:15,62,0.967741935
+1332,1,20130417,1,10:15:16,66,0.909090909
+1333,1,20130417,1,10:15:17,64,0.9375
+1334,1,20130417,1,10:15:18,55,1.090909091
+1335,1,20130417,1,10:15:19,62,0.967741935
+1336,1,20130417,1,10:15:20,71,0.845070423
+1337,1,20130417,1,10:15:21,66,0.909090909
+1338,1,20130417,1,10:15:22,63,0.952380952
+1339,1,20130417,1,10:15:23,69,0.869565217
+1340,1,20130417,1,10:15:24,68,0.882352941
+1341,1,20130417,1,10:15:24,65,0.923076923
+1342,1,20130417,1,10:15:25,68,0.882352941
+1343,1,20130417,1,10:15:26,71,0.845070423
+1344,1,20130417,1,10:15:27,60,1
+1345,1,20130417,1,10:15:28,59,1.016949153
+1346,1,20130417,1,10:15:29,66,0.909090909
+1347,1,20130417,1,10:15:30,63,0.952380952
+1348,1,20130417,1,10:15:31,60,1
+1349,1,20130417,1,10:15:32,65,0.923076923
+1350,1,20130417,1,10:15:33,64,0.9375
+1351,1,20130417,1,10:15:34,60,1
+1352,1,20130417,1,10:15:35,63,0.952380952
+1353,1,20130417,1,10:15:36,69,0.869565217
+1354,1,20130417,1,10:15:37,62,0.967741935
+1355,1,20130417,1,10:15:38,58,1.034482759
+1356,1,20130417,1,10:15:39,64,0.9375
+1357,1,20130417,1,10:15:40,71,0.845070423
+1358,1,20130417,1,10:15:40,66,0.909090909
+1359,1,20130417,1,10:15:41,61,0.983606557
+1360,1,20130417,1,10:15:42,64,0.9375
+1361,1,20130417,1,10:15:43,69,0.869565217
+1362,1,20130417,1,10:15:44,62,0.967741935
+1363,1,20130417,1,10:15:45,59,1.016949153
+1364,1,20130417,1,10:15:46,62,0.967741935
+1365,1,20130417,1,10:15:47,70,0.857142857
+1366,1,20130417,1,10:15:48,63,0.952380952
+1367,1,20130417,1,10:15:49,62,0.967741935
+1368,1,20130417,1,10:15:50,71,0.845070423
+1369,1,20130417,1,10:15:51,76,0.789473684
+1370,1,20130417,1,10:15:52,68,0.882352941
+1371,1,20130417,1,10:15:52,66,0.909090909
+1372,1,20130417,1,10:15:53,69,0.869565217
+1373,1,20130417,1,10:15:54,69,0.869565217
+1374,1,20130417,1,10:15:55,62,0.967741935
+1375,1,20130417,1,10:15:56,62,0.967741935
+1376,1,20130417,1,10:15:57,69,0.869565217
+1377,1,20130417,1,10:15:58,73,0.821917808
+1378,1,20130417,1,10:15:59,71,0.845070423
+1379,1,20130417,1,10:16:00,68,0.882352941
+1380,1,20130417,1,10:16:00,70,0.857142857
+1381,1,20130417,1,10:16:01,68,0.882352941
+1382,1,20130417,1,10:16:02,61,0.983606557
+1383,1,20130417,1,10:16:03,63,0.952380952
+1384,1,20130417,1,10:16:04,70,0.857142857
+1385,1,20130417,1,10:16:05,67,0.895522388
+1386,1,20130417,1,10:16:06,63,0.952380952
+1387,1,20130417,1,10:16:07,65,0.923076923
+1388,1,20130417,1,10:16:08,75,0.8
+1389,1,20130417,1,10:16:09,72,0.833333333
+1390,1,20130417,1,10:16:09,71,0.845070423
+1391,1,20130417,1,10:16:10,76,0.789473684
+1392,1,20130417,1,10:16:11,70,0.857142857
+1393,1,20130417,1,10:16:12,66,0.909090909
+1394,1,20130417,1,10:16:13,74,0.810810811
+1395,1,20130417,1,10:16:14,75,0.8
+1396,1,20130417,1,10:16:14,66,0.909090909
+1397,1,20130417,1,10:16:15,67,0.895522388
+1398,1,20130417,1,10:16:16,70,0.857142857
+1399,1,20130417,1,10:16:17,77,0.779220779
+1400,1,20130417,1,10:16:18,72,0.833333333
+1401,1,20130417,1,10:16:19,64,0.9375
+1402,1,20130417,1,10:16:20,66,0.909090909
+1403,1,20130417,1,10:16:20,75,0.8
+1404,1,20130417,1,10:16:21,72,0.833333333
+1405,1,20130417,1,10:16:22,70,0.857142857
+1406,1,20130417,1,10:16:23,68,0.882352941
+1407,1,20130417,1,10:16:24,76,0.789473684
+1408,1,20130417,1,10:16:25,77,0.779220779
+1409,1,20130417,1,10:16:25,72,0.833333333
+1410,1,20130417,1,10:16:26,75,0.8
+1411,1,20130417,1,10:16:27,78,0.769230769
+1412,1,20130417,1,10:16:28,82,0.731707317
+1413,1,20130417,1,10:16:29,81,0.740740741
+1414,1,20130417,1,10:16:29,67,0.895522388
+1415,1,20130417,1,10:16:30,76,0.789473684
+1416,1,20130417,1,10:16:31,76,0.789473684
+1417,1,20130417,1,10:16:32,80,0.75
+1418,1,20130417,1,10:16:33,79,0.759493671
+1419,1,20130417,1,10:16:33,79,0.759493671
+1420,1,20130417,1,10:16:34,79,0.759493671
+1421,1,20130417,1,10:16:35,72,0.833333333
+1422,1,20130417,1,10:16:36,65,0.923076923
+1423,1,20130417,1,10:16:37,72,0.833333333
+1424,1,20130417,1,10:16:37,73,0.821917808
+1425,1,20130417,1,10:16:38,67,0.895522388
+1426,1,20130417,1,10:16:39,69,0.869565217
+1427,1,20130417,1,10:16:40,76,0.789473684
+1428,1,20130417,1,10:16:41,73,0.821917808
+1429,1,20130417,1,10:16:42,67,0.895522388
+1430,1,20130417,1,10:16:43,66,0.909090909
+1431,1,20130417,1,10:16:43,73,0.821917808
+1432,1,20130417,1,10:16:44,65,0.923076923
+1433,1,20130417,1,10:16:45,65,0.923076923
+1434,1,20130417,1,10:16:46,71,0.845070423
+1435,1,20130417,1,10:16:47,71,0.845070423
+1436,1,20130417,1,10:16:48,65,0.923076923
+1437,1,20130417,1,10:16:49,62,0.967741935
+1438,1,20130417,1,10:16:50,73,0.821917808
+1439,1,20130417,1,10:16:50,68,0.882352941
+1440,1,20130417,1,10:16:51,65,0.923076923
+1441,1,20130417,1,10:16:52,67,0.895522388
+1442,1,20130417,1,10:16:53,73,0.821917808
+1443,1,20130417,1,10:16:54,64,0.9375
+1444,1,20130417,1,10:16:55,60,1
+1445,1,20130417,1,10:16:56,65,0.923076923
+1446,1,20130417,1,10:16:57,63,0.952380952
+1447,1,20130417,1,10:16:58,69,0.869565217
+1448,1,20130417,1,10:16:59,61,0.983606557
+1449,1,20130417,1,10:17:00,60,1
+1450,1,20130417,1,10:17:01,70,0.857142857
+1451,1,20130417,1,10:17:02,68,0.882352941
+1452,1,20130417,1,10:17:02,67,0.895522388
+1453,1,20130417,1,10:17:03,76,0.789473684
+1454,1,20130417,1,10:17:04,76,0.789473684
+1455,1,20130417,1,10:17:05,72,0.833333333
+1456,1,20130417,1,10:17:06,73,0.821917808
+1457,1,20130417,1,10:17:07,71,0.845070423
+1458,1,20130417,1,10:17:07,67,0.895522388
+1459,1,20130417,1,10:17:08,59,1.016949153
+1460,1,20130417,1,10:17:09,70,0.857142857
+1461,1,20130417,1,10:17:10,66,0.909090909
+1462,1,20130417,1,10:17:11,63,0.952380952
+1463,1,20130417,1,10:17:12,64,0.9375
+1464,1,20130417,1,10:17:13,70,0.857142857
+1465,1,20130417,1,10:17:14,68,0.882352941
+1466,1,20130417,1,10:17:15,69,0.869565217
+1467,1,20130417,1,10:17:16,70,0.857142857
+1468,1,20130417,1,10:17:16,76,0.789473684
+1469,1,20130417,1,10:17:17,75,0.8
+1470,1,20130417,1,10:17:18,77,0.779220779
+1471,1,20130417,1,10:17:19,71,0.845070423
+1472,1,20130417,1,10:17:20,69,0.869565217
+1473,1,20130417,1,10:17:21,78,0.769230769
+1474,1,20130417,1,10:17:21,77,0.779220779
+1475,1,20130417,1,10:17:22,70,0.857142857
+1476,1,20130417,1,10:17:23,69,0.869565217
+1477,1,20130417,1,10:17:24,77,0.779220779
+1478,1,20130417,1,10:17:25,82,0.731707317
+1479,1,20130417,1,10:17:25,85,0.705882353
+1480,1,20130417,1,10:17:26,77,0.779220779
+1481,1,20130417,1,10:17:27,72,0.833333333
+1482,1,20130417,1,10:17:28,72,0.833333333
+1483,1,20130417,1,10:17:28,69,0.869565217
+1484,1,20130417,1,10:17:29,67,0.895522388
+1485,1,20130417,1,10:17:30,67,0.895522388
+1486,1,20130417,1,10:17:31,68,0.882352941
+1487,1,20130417,1,10:17:32,68,0.882352941
+1488,1,20130417,1,10:17:33,74,0.810810811
+1489,1,20130417,1,10:17:34,78,0.769230769
+1490,1,20130417,1,10:17:34,74,0.810810811
+1491,1,20130417,1,10:17:35,69,0.869565217
+1492,1,20130417,1,10:17:36,68,0.882352941
+1493,1,20130417,1,10:17:37,72,0.833333333
+1494,1,20130417,1,10:17:38,67,0.895522388
+1495,1,20130417,1,10:17:39,60,1
+1496,1,20130417,1,10:17:40,58,1.034482759
+1497,1,20130417,1,10:17:41,59,1.016949153
+1498,1,20130417,1,10:17:42,66,0.909090909
+1499,1,20130417,1,10:17:43,68,0.882352941
+1500,1,20130417,1,10:17:44,65,0.923076923
+1501,1,20130417,1,10:17:45,67,0.895522388
+1502,1,20130417,1,10:17:45,72,0.833333333
+1503,1,20130417,1,10:17:46,66,0.909090909
+1504,1,20130417,1,10:17:47,62,0.967741935
+1505,1,20130417,1,10:17:48,65,0.923076923
+1506,1,20130417,1,10:17:49,72,0.833333333
+1507,1,20130417,1,10:17:50,68,0.882352941
+1508,1,20130417,1,10:17:51,67,0.895522388
+1509,1,20130417,1,10:17:52,69,0.869565217
+1510,1,20130417,1,10:17:53,71,0.845070423
+1511,1,20130417,1,10:17:53,79,0.759493671
+1512,1,20130417,1,10:17:54,75,0.8
+1513,1,20130417,1,10:17:55,60,1
+1514,1,20130417,1,10:17:56,59,1.016949153
+1515,1,20130417,1,10:17:57,67,0.895522388
+1516,1,20130417,1,10:17:58,70,0.857142857
+1517,1,20130417,1,10:17:59,66,0.909090909
+1518,1,20130417,1,10:18:00,65,0.923076923
+1519,1,20130417,1,10:18:01,67,0.895522388
+1520,1,20130417,1,10:18:01,71,0.845070423
+1521,1,20130417,1,10:18:02,70,0.857142857
+1522,1,20130417,1,10:18:03,75,0.8
+1523,1,20130417,1,10:18:04,77,0.779220779
+1524,1,20130417,1,10:18:05,78,0.769230769
+1525,1,20130417,1,10:18:05,68,0.882352941
+1526,1,20130417,1,10:18:06,65,0.923076923
+1527,1,20130417,1,10:18:07,67,0.895522388
+1528,1,20130417,1,10:18:08,66,0.909090909
+1529,1,20130417,1,10:18:09,65,0.923076923
+1530,1,20130417,1,10:18:10,67,0.895522388
+1531,1,20130417,1,10:18:11,72,0.833333333
+1532,1,20130417,1,10:18:12,69,0.869565217
+1533,1,20130417,1,10:18:13,65,0.923076923
+1534,1,20130417,1,10:18:14,74,0.810810811
+1535,1,20130417,1,10:18:14,73,0.821917808
+1536,1,20130417,1,10:18:15,69,0.869565217
+1537,1,20130417,1,10:18:16,63,0.952380952
+1538,1,20130417,1,10:18:17,70,0.857142857
+1539,1,20130417,1,10:18:18,68,0.882352941
+1540,1,20130417,1,10:18:19,63,0.952380952
+1541,1,20130417,1,10:18:20,65,0.923076923
+1542,1,20130417,1,10:18:21,70,0.857142857
+1543,1,20130417,1,10:18:21,68,0.882352941
+1544,1,20130417,1,10:18:22,66,0.909090909
+1545,1,20130417,1,10:18:23,71,0.845070423
+1546,1,20130417,1,10:18:24,73,0.821917808
+1547,1,20130417,1,10:18:25,63,0.952380952
+1548,1,20130417,1,10:18:26,61,0.983606557
+1549,1,20130417,1,10:18:27,70,0.857142857
+1550,1,20130417,1,10:18:28,69,0.869565217
+1551,1,20130417,1,10:18:29,61,0.983606557
+1552,1,20130417,1,10:18:30,65,0.923076923
+1553,1,20130417,1,10:18:30,71,0.845070423
+1554,1,20130417,1,10:18:31,65,0.923076923
+1555,1,20130417,1,10:18:32,60,1
+1556,1,20130417,1,10:18:33,61,0.983606557
+1557,1,20130417,1,10:18:34,66,0.909090909
+1558,1,20130417,1,10:18:35,65,0.923076923
+1559,1,20130417,1,10:18:36,67,0.895522388
+1560,1,20130417,1,10:18:37,71,0.845070423
+1561,1,20130417,1,10:18:38,69,0.869565217
+1562,1,20130417,1,10:18:39,62,0.967741935
+1563,1,20130417,1,10:18:40,70,0.857142857
+1564,1,20130417,1,10:18:41,66,0.909090909
+1565,1,20130417,1,10:18:41,63,0.952380952
+1566,1,20130417,1,10:18:42,70,0.857142857
+1567,1,20130417,1,10:18:43,77,0.779220779
+1568,1,20130417,1,10:18:44,75,0.8
+1569,1,20130417,1,10:18:45,70,0.857142857
+1570,1,20130417,1,10:18:46,66,0.909090909
+1571,1,20130417,1,10:18:47,73,0.821917808
+1572,1,20130417,1,10:18:47,69,0.869565217
+1573,1,20130417,1,10:18:48,66,0.909090909
+1574,1,20130417,1,10:18:49,72,0.833333333
+1575,1,20130417,1,10:18:50,73,0.821917808
+1576,1,20130417,1,10:18:51,67,0.895522388
+1577,1,20130417,1,10:18:52,67,0.895522388
+1578,1,20130417,1,10:18:53,74,0.810810811
+1579,1,20130417,1,10:18:53,69,0.869565217
+1580,1,20130417,1,10:18:54,65,0.923076923
+1581,1,20130417,1,10:18:55,69,0.869565217
+1582,1,20130417,1,10:18:56,78,0.769230769
+1583,1,20130417,1,10:18:57,78,0.769230769
+1584,1,20130417,1,10:18:58,81,0.740740741
+1585,1,20130417,1,10:18:58,83,0.722891566
+1586,1,20130417,1,10:18:59,89,0.674157303
+1587,1,20130417,1,10:19:00,90,0.666666667
+1588,1,20130417,1,10:19:00,82,0.731707317
+1589,1,20130417,1,10:19:01,76,0.789473684
+1590,1,20130417,1,10:19:02,75,0.8
+1591,1,20130417,1,10:19:03,81,0.740740741
+1592,1,20130417,1,10:19:03,78,0.769230769
+1593,1,20130417,1,10:19:04,74,0.810810811
+1594,1,20130417,1,10:19:05,73,0.821917808
+1595,1,20130417,1,10:19:06,78,0.769230769
+1596,1,20130417,1,10:19:07,74,0.810810811
+1597,1,20130417,1,10:19:07,70,0.857142857
+1598,1,20130417,1,10:19:08,70,0.857142857
+1599,1,20130417,1,10:19:09,76,0.789473684
+1600,1,20130417,1,10:19:10,78,0.769230769
+1601,1,20130417,1,10:19:11,75,0.8
+1602,1,20130417,1,10:19:12,72,0.833333333
+1603,1,20130417,1,10:19:12,71,0.845070423
+1604,1,20130417,1,10:19:13,77,0.779220779
+1605,1,20130417,1,10:19:14,67,0.895522388
+1606,1,20130417,1,10:19:15,60,1
+1607,1,20130417,1,10:19:16,63,0.952380952
+1608,1,20130417,1,10:19:17,71,0.845070423
+1609,1,20130417,1,10:19:18,69,0.869565217
+1610,1,20130417,1,10:19:19,66,0.909090909
+1611,1,20130417,1,10:19:19,63,0.952380952
+1612,1,20130417,1,10:19:20,72,0.833333333
+1613,1,20130417,1,10:19:21,70,0.857142857
+1614,1,20130417,1,10:19:22,66,0.909090909
+1615,1,20130417,1,10:19:23,68,0.882352941
+1616,1,20130417,1,10:19:24,75,0.8
+1617,1,20130417,1,10:19:25,70,0.857142857
+1618,1,20130417,1,10:19:26,67,0.895522388
+1619,1,20130417,1,10:19:26,71,0.845070423
+1620,1,20130417,1,10:19:27,73,0.821917808
+1621,1,20130417,1,10:19:28,71,0.845070423
+1622,1,20130417,1,10:19:29,63,0.952380952
+1623,1,20130417,1,10:19:30,69,0.869565217
+1624,1,20130417,1,10:19:31,71,0.845070423
+1625,1,20130417,1,10:19:32,61,0.983606557
+1626,1,20130417,1,10:19:33,63,0.952380952
+1627,1,20130417,1,10:19:34,70,0.857142857
+1628,1,20130417,1,10:19:34,68,0.882352941
+1629,1,20130417,1,10:19:35,73,0.821917808
+1630,1,20130417,1,10:19:36,76,0.789473684
+1631,1,20130417,1,10:19:37,74,0.810810811
+1632,1,20130417,1,10:19:38,68,0.882352941
+1633,1,20130417,1,10:19:39,71,0.845070423
+1634,1,20130417,1,10:19:39,70,0.857142857
+1635,1,20130417,1,10:19:40,64,0.9375
+1636,1,20130417,1,10:19:41,63,0.952380952
+1637,1,20130417,1,10:19:42,73,0.821917808
+1638,1,20130417,1,10:19:43,70,0.857142857
+1639,1,20130417,1,10:19:44,69,0.869565217
+1640,1,20130417,1,10:19:45,74,0.810810811
+1641,1,20130417,1,10:19:46,78,0.769230769
+1642,1,20130417,1,10:19:46,72,0.833333333
+1643,1,20130417,1,10:19:47,67,0.895522388
+1644,1,20130417,1,10:19:48,71,0.845070423
+1645,1,20130417,1,10:19:49,74,0.810810811
+1646,1,20130417,1,10:19:50,69,0.869565217
+1647,1,20130417,1,10:19:51,68,0.882352941
+1648,1,20130417,1,10:19:51,73,0.821917808
+1649,1,20130417,1,10:19:52,73,0.821917808
+1650,1,20130417,1,10:19:53,69,0.869565217
+1651,1,20130417,1,10:19:54,68,0.882352941
+1652,1,20130417,1,10:19:55,74,0.810810811
+1653,1,20130417,1,10:19:56,73,0.821917808
+1654,1,20130417,1,10:19:57,69,0.869565217
+1655,1,20130417,1,10:19:57,67,0.895522388
+1656,1,20130417,1,10:19:58,75,0.8
+1657,1,20130417,1,10:19:59,72,0.833333333
+1658,1,20130417,1,10:20:00,71,0.845070423
+1659,1,20130417,1,10:20:01,70,0.857142857
+1660,1,20130417,1,10:20:02,79,0.759493671
+1661,1,20130417,1,10:20:02,81,0.740740741
+1662,1,20130417,1,10:20:03,78,0.769230769
+1663,1,20130417,1,10:20:04,77,0.779220779
+1664,1,20130417,1,10:20:05,80,0.75
+1665,1,20130417,1,10:20:05,85,0.705882353
+1666,1,20130417,1,10:20:06,84,0.714285714
+1667,1,20130417,1,10:20:07,69,0.869565217
+1668,1,20130417,1,10:20:08,68,0.882352941
+1669,1,20130417,1,10:20:09,76,0.789473684
+1670,1,20130417,1,10:20:09,74,0.810810811
+1671,1,20130417,1,10:20:10,68,0.882352941
+1672,1,20130417,1,10:20:11,78,0.769230769
+1673,1,20130417,1,10:20:12,79,0.759493671
+1674,1,20130417,1,10:20:13,77,0.779220779
+1675,1,20130417,1,10:20:13,83,0.722891566
+1676,1,20130417,1,10:20:14,80,0.75
+1677,1,20130417,1,10:20:15,74,0.810810811
+1678,1,20130417,1,10:20:16,69,0.869565217
+1679,1,20130417,1,10:20:17,78,0.769230769
+1680,1,20130417,1,10:20:17,82,0.731707317
+1681,1,20130417,1,10:20:18,79,0.759493671
+1682,1,20130417,1,10:20:19,76,0.789473684
+1683,1,20130417,1,10:20:20,71,0.845070423
+1684,1,20130417,1,10:20:20,76,0.789473684
+1685,1,20130417,1,10:20:21,73,0.821917808
+1686,1,20130417,1,10:20:22,80,0.75
+1687,1,20130417,1,10:20:23,80,0.75
+1688,1,20130417,1,10:20:24,77,0.779220779
+1689,1,20130417,1,10:20:24,75,0.8
+1690,1,20130417,1,10:20:25,75,0.8
+1691,1,20130417,1,10:20:26,83,0.722891566
+1692,1,20130417,1,10:20:27,85,0.705882353
+1693,1,20130417,1,10:20:27,84,0.714285714
+1694,1,20130417,1,10:20:28,82,0.731707317
+1695,1,20130417,1,10:20:29,84,0.714285714
+1696,1,20130417,1,10:20:30,86,0.697674419
+1697,1,20130417,1,10:20:30,79,0.759493671
+1698,1,20130417,1,10:20:31,67,0.895522388
+1699,1,20130417,1,10:20:32,65,0.923076923
+1700,1,20130417,1,10:20:33,70,0.857142857
+1701,1,20130417,1,10:20:34,72,0.833333333
+1702,1,20130417,1,10:20:34,66,0.909090909
+1703,1,20130417,1,10:20:35,68,0.882352941
+1704,1,20130417,1,10:20:36,68,0.882352941
+1705,1,20130417,1,10:20:37,74,0.810810811
+1706,1,20130417,1,10:20:38,70,0.857142857
+1707,1,20130417,1,10:20:39,62,0.967741935
+1708,1,20130417,1,10:20:40,66,0.909090909
+1709,1,20130417,1,10:20:41,72,0.833333333
+1710,1,20130417,1,10:20:42,63,0.952380952
+1711,1,20130417,1,10:20:42,63,0.952380952
+1712,1,20130417,1,10:20:43,68,0.882352941
+1713,1,20130417,1,10:20:44,66,0.909090909
+1714,1,20130417,1,10:20:45,66,0.909090909
+1715,1,20130417,1,10:20:46,65,0.923076923
+1716,1,20130417,1,10:20:47,70,0.857142857
+1717,1,20130417,1,10:20:48,71,0.845070423
+1718,1,20130417,1,10:20:49,65,0.923076923
+1719,1,20130417,1,10:20:50,63,0.952380952
+1720,1,20130417,1,10:20:51,73,0.821917808
+1721,1,20130417,1,10:20:51,68,0.882352941
+1722,1,20130417,1,10:20:52,65,0.923076923
+1723,1,20130417,1,10:20:53,67,0.895522388
+1724,1,20130417,1,10:20:54,77,0.779220779
+1725,1,20130417,1,10:20:55,73,0.821917808
+1726,1,20130417,1,10:20:56,66,0.909090909
+1727,1,20130417,1,10:20:57,62,0.967741935
+1728,1,20130417,1,10:20:58,67,0.895522388
+1729,1,20130417,1,10:20:59,68,0.882352941
+1730,1,20130417,1,10:20:59,66,0.909090909
+1731,1,20130417,1,10:21:00,67,0.895522388
+1732,1,20130417,1,10:21:01,73,0.821917808
+1733,1,20130417,1,10:21:02,69,0.869565217
+1734,1,20130417,1,10:21:03,65,0.923076923
+1735,1,20130417,1,10:21:04,67,0.895522388
+1736,1,20130417,1,10:21:05,75,0.8
+1737,1,20130417,1,10:21:06,71,0.845070423
+1738,1,20130417,1,10:21:06,68,0.882352941
+1739,1,20130417,1,10:21:07,67,0.895522388
+1740,1,20130417,1,10:21:08,76,0.789473684
+1741,1,20130417,1,10:21:09,72,0.833333333
+1742,1,20130417,1,10:21:10,69,0.869565217
+1743,1,20130417,1,10:21:11,73,0.821917808
+1744,1,20130417,1,10:21:11,77,0.779220779
+1745,1,20130417,1,10:21:12,70,0.857142857
+1746,1,20130417,1,10:21:13,65,0.923076923
+1747,1,20130417,1,10:21:14,70,0.857142857
+1748,1,20130417,1,10:21:15,71,0.845070423
+1749,1,20130417,1,10:21:16,69,0.869565217
+1750,1,20130417,1,10:21:17,72,0.833333333
+1751,1,20130417,1,10:21:17,79,0.759493671
+1752,1,20130417,1,10:21:18,79,0.759493671
+1753,1,20130417,1,10:21:19,76,0.789473684
+1754,1,20130417,1,10:21:20,71,0.845070423
+1755,1,20130417,1,10:21:21,78,0.769230769
+1756,1,20130417,1,10:21:21,79,0.759493671
+1757,1,20130417,1,10:21:22,73,0.821917808
+1758,1,20130417,1,10:21:23,67,0.895522388
+1759,1,20130417,1,10:21:24,77,0.779220779
+1760,1,20130417,1,10:21:25,74,0.810810811
+1761,1,20130417,1,10:21:25,71,0.845070423
+1762,1,20130417,1,10:21:26,69,0.869565217
+1763,1,20130417,1,10:21:27,76,0.789473684
+1764,1,20130417,1,10:21:28,73,0.821917808
+1765,1,20130417,1,10:21:29,70,0.857142857
+1766,1,20130417,1,10:21:30,69,0.869565217
+1767,1,20130417,1,10:21:30,76,0.789473684
+1768,1,20130417,1,10:21:31,74,0.810810811
+1769,1,20130417,1,10:21:32,72,0.833333333
+1770,1,20130417,1,10:21:33,66,0.909090909
+1771,1,20130417,1,10:21:34,73,0.821917808
+1772,1,20130417,1,10:21:35,68,0.882352941
+1773,1,20130417,1,10:21:36,69,0.869565217
+1774,1,20130417,1,10:21:36,65,0.923076923
+1775,1,20130417,1,10:21:37,76,0.789473684
+1776,1,20130417,1,10:21:38,80,0.75
+1777,1,20130417,1,10:21:39,80,0.75
+1778,1,20130417,1,10:21:40,80,0.75
+1779,1,20130417,1,10:21:40,83,0.722891566
+1780,1,20130417,1,10:21:41,85,0.705882353
+1781,1,20130417,1,10:21:42,80,0.75
+1782,1,20130417,1,10:21:43,84,0.714285714
+1783,1,20130417,1,10:21:43,83,0.722891566
+1784,1,20130417,1,10:21:44,78,0.769230769
+1785,1,20130417,1,10:21:45,73,0.821917808
+1786,1,20130417,1,10:21:46,75,0.8
+1787,1,20130417,1,10:21:46,78,0.769230769
+1788,1,20130417,1,10:21:47,75,0.8
+1789,1,20130417,1,10:21:48,74,0.810810811
+1790,1,20130417,1,10:21:49,77,0.779220779
+1791,1,20130417,1,10:21:50,84,0.714285714
+1792,1,20130417,1,10:21:50,87,0.689655172
+1793,1,20130417,1,10:21:51,82,0.731707317
+1794,1,20130417,1,10:21:52,73,0.821917808
+1795,1,20130417,1,10:21:53,71,0.845070423
+1796,1,20130417,1,10:21:53,78,0.769230769
+1797,1,20130417,1,10:21:54,72,0.833333333
+1798,1,20130417,1,10:21:55,67,0.895522388
+1799,1,20130417,1,10:21:56,67,0.895522388
+1800,1,20130417,1,10:21:57,75,0.8
+1801,1,20130417,1,10:21:58,77,0.779220779
+1802,1,20130417,1,10:21:58,70,0.857142857
+1803,1,20130417,1,10:21:59,69,0.869565217
+1804,1,20130417,1,10:22:00,73,0.821917808
+1805,1,20130417,1,10:22:01,73,0.821917808
+1806,1,20130417,1,10:22:02,64,0.9375
+1807,1,20130417,1,10:22:03,64,0.9375
+1808,1,20130417,1,10:22:04,67,0.895522388
+1809,1,20130417,1,10:22:04,73,0.821917808
+1810,1,20130417,1,10:22:05,71,0.845070423
+1811,1,20130417,1,10:22:06,70,0.857142857
+1812,1,20130417,1,10:22:07,68,0.882352941
+1813,1,20130417,1,10:22:08,72,0.833333333
+1814,1,20130417,1,10:22:09,72,0.833333333
+1815,1,20130417,1,10:22:10,65,0.923076923
+1816,1,20130417,1,10:22:10,60,1
+1817,1,20130417,1,10:22:11,63,0.952380952
+1818,1,20130417,1,10:22:12,73,0.821917808
+1819,1,20130417,1,10:22:13,68,0.882352941
+1820,1,20130417,1,10:22:14,63,0.952380952
+1821,1,20130417,1,10:22:15,64,0.9375
+1822,1,20130417,1,10:22:16,72,0.833333333
+1823,1,20130417,1,10:22:17,72,0.833333333
+1824,1,20130417,1,10:22:18,68,0.882352941
+1825,1,20130417,1,10:22:19,64,0.9375
+1826,1,20130417,1,10:22:19,71,0.845070423
+1827,1,20130417,1,10:22:20,69,0.869565217
+1828,1,20130417,1,10:22:21,70,0.857142857
+1829,1,20130417,1,10:22:22,74,0.810810811
+1830,1,20130417,1,10:22:23,82,0.731707317
+1831,1,20130417,1,10:22:24,81,0.740740741
+1832,1,20130417,1,10:22:24,75,0.8
+1833,1,20130417,1,10:22:25,72,0.833333333
+1834,1,20130417,1,10:22:26,72,0.833333333
+1835,1,20130417,1,10:22:27,82,0.731707317
+1836,1,20130417,1,10:22:28,82,0.731707317
+1837,1,20130417,1,10:22:28,80,0.75
+1838,1,20130417,1,10:22:29,80,0.75
+1839,1,20130417,1,10:22:30,85,0.705882353
+1840,1,20130417,1,10:22:30,84,0.714285714
+1841,1,20130417,1,10:22:31,78,0.769230769
+1842,1,20130417,1,10:22:32,71,0.845070423
+1843,1,20130417,1,10:22:33,71,0.845070423
+1844,1,20130417,1,10:22:34,79,0.759493671
+1845,1,20130417,1,10:22:34,77,0.779220779
+1846,1,20130417,1,10:22:35,73,0.821917808
+1847,1,20130417,1,10:22:36,72,0.833333333
+1848,1,20130417,1,10:22:37,74,0.810810811
+1849,1,20130417,1,10:22:38,79,0.759493671
+1850,1,20130417,1,10:22:38,77,0.779220779
+1851,1,20130417,1,10:22:39,76,0.789473684
+1852,1,20130417,1,10:22:40,76,0.789473684
+1853,1,20130417,1,10:22:41,81,0.740740741
+1854,1,20130417,1,10:22:42,78,0.769230769
+1855,1,20130417,1,10:22:42,69,0.869565217
+1856,1,20130417,1,10:22:43,66,0.909090909
+1857,1,20130417,1,10:22:44,67,0.895522388
+1858,1,20130417,1,10:22:45,75,0.8
+1859,1,20130417,1,10:22:46,73,0.821917808
+1860,1,20130417,1,10:22:47,71,0.845070423
+1861,1,20130417,1,10:22:47,70,0.857142857
+1862,1,20130417,1,10:22:48,73,0.821917808
+1863,1,20130417,1,10:22:49,77,0.779220779
+1864,1,20130417,1,10:22:50,68,0.882352941
+1865,1,20130417,1,10:22:51,62,0.967741935
+1866,1,20130417,1,10:22:52,62,0.967741935
+1867,1,20130417,1,10:22:53,72,0.833333333
+1868,1,20130417,1,10:22:54,67,0.895522388
+1869,1,20130417,1,10:22:54,63,0.952380952
+1870,1,20130417,1,10:22:55,64,0.9375
+1871,1,20130417,1,10:22:56,71,0.845070423
+1872,1,20130417,1,10:22:57,68,0.882352941
+1873,1,20130417,1,10:22:58,64,0.9375
+1874,1,20130417,1,10:22:59,62,0.967741935
+1875,1,20130417,1,10:23:00,72,0.833333333
+1876,1,20130417,1,10:23:01,71,0.845070423
+1877,1,20130417,1,10:23:02,69,0.869565217
+1878,1,20130417,1,10:23:02,69,0.869565217
+1879,1,20130417,1,10:23:03,74,0.810810811
+1880,1,20130417,1,10:23:04,68,0.882352941
+1881,1,20130417,1,10:23:05,61,0.983606557
+1882,1,20130417,1,10:23:06,60,1
+1883,1,20130417,1,10:23:07,71,0.845070423
+1884,1,20130417,1,10:23:08,66,0.909090909
+1885,1,20130417,1,10:23:09,72,0.833333333
+1886,1,20130417,1,10:23:10,79,0.759493671
+1887,1,20130417,1,10:23:10,73,0.821917808
+1888,1,20130417,1,10:23:11,68,0.882352941
+1889,1,20130417,1,10:23:12,58,1.034482759
+1890,1,20130417,1,10:23:13,67,0.895522388
+1891,1,20130417,1,10:23:14,64,0.9375
+1892,1,20130417,1,10:23:15,61,0.983606557
+1893,1,20130417,1,10:23:16,67,0.895522388
+1894,1,20130417,1,10:23:17,68,0.882352941
+1895,1,20130417,1,10:23:18,59,1.016949153
+1896,1,20130417,1,10:23:19,64,0.9375
+1897,1,20130417,1,10:23:20,75,0.8
+1898,1,20130417,1,10:23:20,69,0.869565217
+1899,1,20130417,1,10:23:21,72,0.833333333
+1900,1,20130417,1,10:23:22,79,0.759493671
+1901,1,20130417,1,10:23:23,77,0.779220779
+1902,1,20130417,1,10:23:24,79,0.759493671
+1903,1,20130417,1,10:23:24,71,0.845070423
+1904,1,20130417,1,10:23:25,67,0.895522388
+1905,1,20130417,1,10:23:26,72,0.833333333
+1906,1,20130417,1,10:23:27,79,0.759493671
+1907,1,20130417,1,10:23:28,79,0.759493671
+1908,1,20130417,1,10:23:29,81,0.740740741
+1909,1,20130417,1,10:23:29,83,0.722891566
+1910,1,20130417,1,10:23:30,85,0.705882353
+1911,1,20130417,1,10:23:31,78,0.769230769
+1912,1,20130417,1,10:23:32,72,0.833333333
+1913,1,20130417,1,10:23:32,70,0.857142857
+1914,1,20130417,1,10:23:33,75,0.8
+1915,1,20130417,1,10:23:34,76,0.789473684
+1916,1,20130417,1,10:23:35,75,0.8
+1917,1,20130417,1,10:23:36,72,0.833333333
+1918,1,20130417,1,10:23:36,71,0.845070423
+1919,1,20130417,1,10:23:37,74,0.810810811
+1920,1,20130417,1,10:23:38,73,0.821917808
+1921,1,20130417,1,10:23:39,70,0.857142857
+1922,1,20130417,1,10:23:40,67,0.895522388
+1923,1,20130417,1,10:23:41,67,0.895522388
+1924,1,20130417,1,10:23:42,71,0.845070423
+1925,1,20130417,1,10:23:42,73,0.821917808
+1926,1,20130417,1,10:23:43,65,0.923076923
+1927,1,20130417,1,10:23:44,61,0.983606557
+1928,1,20130417,1,10:23:45,64,0.9375
+1929,1,20130417,1,10:23:46,70,0.857142857
+1930,1,20130417,1,10:23:47,73,0.821917808
+1931,1,20130417,1,10:23:48,70,0.857142857
+1932,1,20130417,1,10:23:49,71,0.845070423
+1933,1,20130417,1,10:23:49,70,0.857142857
+1934,1,20130417,1,10:23:50,64,0.9375
+1935,1,20130417,1,10:23:51,57,1.052631579
+1936,1,20130417,1,10:23:52,61,0.983606557
+1937,1,20130417,1,10:23:53,72,0.833333333
+1938,1,20130417,1,10:23:54,73,0.821917808
+1939,1,20130417,1,10:23:55,82,0.731707317
+1940,1,20130417,1,10:23:56,83,0.722891566
+1941,1,20130417,1,10:23:56,83,0.722891566
+1942,1,20130417,1,10:23:57,87,0.689655172
+1943,1,20130417,1,10:23:58,93,0.64516129
+1944,1,20130417,1,10:23:58,90,0.666666667
+1945,1,20130417,1,10:23:59,93,0.64516129
+1946,1,20130417,1,10:24:00,94,0.638297872
+1947,1,20130417,1,10:24:00,94,0.638297872
+1948,1,20130417,1,10:24:01,95,0.631578947
+1949,1,20130417,1,10:24:02,97,0.618556701
+1950,1,20130417,1,10:24:02,93,0.64516129
+1951,1,20130417,1,10:24:03,87,0.689655172
+1952,1,20130417,1,10:24:04,82,0.731707317
+1953,1,20130417,1,10:24:04,82,0.731707317
+1954,1,20130417,1,10:24:05,88,0.681818182
+1955,1,20130417,1,10:24:06,89,0.674157303
+1956,1,20130417,1,10:24:06,87,0.689655172
+1957,1,20130417,1,10:24:07,87,0.689655172
+1958,1,20130417,1,10:24:08,75,0.8
+1959,1,20130417,1,10:24:09,71,0.845070423
+1960,1,20130417,1,10:24:09,68,0.882352941
+1961,1,20130417,1,10:24:10,72,0.833333333
+1962,1,20130417,1,10:24:11,65,0.923076923
+1963,1,20130417,1,10:24:12,57,1.052631579
+1964,1,20130417,1,10:24:13,59,1.016949153
+1965,1,20130417,1,10:24:14,70,0.857142857
+1966,1,20130417,1,10:24:15,65,0.923076923
+1967,1,20130417,1,10:24:16,61,0.983606557
+1968,1,20130417,1,10:24:17,61,0.983606557
+1969,1,20130417,1,10:24:18,72,0.833333333
+1970,1,20130417,1,10:24:19,64,0.9375
+1971,1,20130417,1,10:24:20,64,0.9375
+1972,1,20130417,1,10:24:21,59,1.016949153
+1973,1,20130417,1,10:24:22,63,0.952380952
+1974,1,20130417,1,10:24:23,57,1.052631579
+1975,1,20130417,1,10:24:24,68,0.882352941
+1976,1,20130417,1,10:24:25,63,0.952380952
+1977,1,20130417,1,10:24:25,73,0.821917808
+1978,1,20130417,1,10:24:26,58,1.034482759
+1979,1,20130417,1,10:24:27,57,1.052631579
+1980,1,20130417,1,10:24:28,63,0.952380952
+1981,1,20130417,1,10:24:29,72,0.833333333
+1982,1,20130417,1,10:24:30,79,0.759493671
+1983,1,20130417,1,10:24:31,79,0.759493671
+1984,1,20130417,1,10:24:32,75,0.8
+1985,1,20130417,1,10:24:32,67,0.895522388
+1986,1,20130417,1,10:24:33,71,0.845070423
+1987,1,20130417,1,10:24:34,65,0.923076923
+1988,1,20130417,1,10:24:35,61,0.983606557
+1989,1,20130417,1,10:24:36,68,0.882352941
+1990,1,20130417,1,10:24:37,69,0.869565217
+1991,1,20130417,1,10:24:38,63,0.952380952
+1992,1,20130417,1,10:24:39,64,0.9375
+1993,1,20130417,1,10:24:40,68,0.882352941
+1994,1,20130417,1,10:24:41,71,0.845070423
+1995,1,20130417,1,10:24:41,61,0.983606557
+1996,1,20130417,1,10:24:42,59,1.016949153
+1997,1,20130417,1,10:24:43,65,0.923076923
+1998,1,20130417,1,10:24:44,65,0.923076923
+1999,1,20130417,1,10:24:45,60,1
+2000,1,20130417,1,10:24:46,60,1
+2001,1,20130417,1,10:24:47,67,0.895522388
+2002,1,20130417,1,10:24:48,64,0.9375
+2003,1,20130417,1,10:24:49,64,0.9375
+2004,1,20130417,1,10:24:50,61,0.983606557
+2005,1,20130417,1,10:24:51,67,0.895522388
+2006,1,20130417,1,10:24:52,67,0.895522388
+2007,1,20130417,1,10:24:53,62,0.967741935
+2008,1,20130417,1,10:24:54,61,0.983606557
+2009,1,20130417,1,10:24:55,69,0.869565217
+2010,1,20130417,1,10:24:56,72,0.833333333
+2011,1,20130417,1,10:24:57,66,0.909090909
+2012,1,20130417,1,10:24:57,66,0.909090909
+2013,1,20130417,1,10:24:58,72,0.833333333
+2014,1,20130417,1,10:24:59,69,0.869565217
+2015,1,20130417,1,10:25:00,64,0.9375
+2016,1,20130417,1,10:25:01,60,1
+2017,1,20130417,1,10:25:02,66,0.909090909
+2018,1,20130417,1,10:25:03,59,1.016949153
+2019,1,20130417,1,10:25:04,61,0.983606557
+2020,1,20130417,1,10:25:05,67,0.895522388
+2021,1,20130417,1,10:25:06,68,0.882352941
+2022,1,20130417,1,10:25:07,66,0.909090909
+2023,1,20130417,1,10:25:08,64,0.9375
+2024,1,20130417,1,10:25:09,67,0.895522388
+2025,1,20130417,1,10:25:09,74,0.810810811
+2026,1,20130417,1,10:25:10,72,0.833333333
+2027,1,20130417,1,10:25:11,63,0.952380952
+2028,1,20130417,1,10:25:12,63,0.952380952
+2029,1,20130417,1,10:25:13,69,0.869565217
+2030,1,20130417,1,10:25:14,65,0.923076923
+2031,1,20130417,1,10:25:15,60,1
+2032,1,20130417,1,10:25:16,69,0.869565217
+2033,1,20130417,1,10:25:17,76,0.789473684
+2034,1,20130417,1,10:25:17,76,0.789473684
+2035,1,20130417,1,10:25:18,72,0.833333333
+2036,1,20130417,1,10:25:19,65,0.923076923
+2037,1,20130417,1,10:25:20,65,0.923076923
+2038,1,20130417,1,10:25:21,77,0.779220779
+2039,1,20130417,1,10:25:22,74,0.810810811
+2040,1,20130417,1,10:25:22,70,0.857142857
+2041,1,20130417,1,10:25:23,70,0.857142857
+2042,1,20130417,1,10:25:24,74,0.810810811
+2043,1,20130417,1,10:25:25,74,0.810810811
+2044,1,20130417,1,10:25:26,72,0.833333333
+2045,1,20130417,1,10:25:27,77,0.779220779
+2046,1,20130417,1,10:25:27,82,0.731707317
+2047,1,20130417,1,10:25:28,75,0.8
+2048,1,20130417,1,10:25:29,64,0.9375
+2049,1,20130417,1,10:25:30,59,1.016949153
+2050,1,20130417,1,10:25:31,62,0.967741935
+2051,1,20130417,1,10:25:32,70,0.857142857
+2052,1,20130417,1,10:25:33,63,0.952380952
+2053,1,20130417,1,10:25:34,60,1
+2054,1,20130417,1,10:25:35,61,0.983606557
+2055,1,20130417,1,10:25:36,69,0.869565217
+2056,1,20130417,1,10:25:37,64,0.9375
+2057,1,20130417,1,10:25:38,59,1.016949153
+2058,1,20130417,1,10:25:39,63,0.952380952
+2059,1,20130417,1,10:25:39,70,0.857142857
+2060,1,20130417,1,10:25:40,73,0.821917808
+2061,1,20130417,1,10:25:41,65,0.923076923
+2062,1,20130417,1,10:25:42,59,1.016949153
+2063,1,20130417,1,10:25:43,60,1
+2064,1,20130417,1,10:25:44,70,0.857142857
+2065,1,20130417,1,10:25:45,64,0.9375
+2066,1,20130417,1,10:25:46,68,0.882352941
+2067,1,20130417,1,10:25:47,77,0.779220779
+2068,1,20130417,1,10:25:48,75,0.8
+2069,1,20130417,1,10:25:48,68,0.882352941
+2070,1,20130417,1,10:25:49,61,0.983606557
+2071,1,20130417,1,10:25:50,72,0.833333333
+2072,1,20130417,1,10:25:51,69,0.869565217
+2073,1,20130417,1,10:25:52,60,1
+2074,1,20130417,1,10:25:53,61,0.983606557
+2075,1,20130417,1,10:25:54,72,0.833333333
+2076,1,20130417,1,10:25:55,78,0.769230769
+2077,1,20130417,1,10:25:56,73,0.821917808
+2078,1,20130417,1,10:25:56,69,0.869565217
+2079,1,20130417,1,10:25:57,66,0.909090909
+2080,1,20130417,1,10:25:58,72,0.833333333
+2081,1,20130417,1,10:25:59,72,0.833333333
+2082,1,20130417,1,10:26:00,61,0.983606557
+2083,1,20130417,1,10:26:01,61,0.983606557
+2084,1,20130417,1,10:26:02,75,0.8
+2085,1,20130417,1,10:26:03,76,0.789473684
+2086,1,20130417,1,10:26:03,74,0.810810811
+2087,1,20130417,1,10:26:04,59,1.016949153
+2088,1,20130417,1,10:26:05,68,0.882352941
+2089,1,20130417,1,10:26:06,58,1.034482759
+2090,1,20130417,1,10:26:07,55,1.090909091
+2091,1,20130417,1,10:26:08,60,1
+2092,1,20130417,1,10:26:09,58,1.034482759
+2093,1,20130417,1,10:26:10,61,0.983606557
+2094,1,20130417,1,10:26:11,64,0.9375
+2095,1,20130417,1,10:26:12,60,1
+2096,1,20130417,1,10:26:13,60,1
+2097,1,20130417,1,10:26:14,60,1
+2098,1,20130417,1,10:26:15,57,1.052631579
+2099,1,20130417,1,10:26:16,59,1.016949153
+2100,1,20130417,1,10:26:17,59,1.016949153
+2101,1,20130417,1,10:26:18,72,0.833333333
+2102,1,20130417,1,10:26:19,66,0.909090909
+2103,1,20130417,1,10:26:20,65,0.923076923
+2104,1,20130417,1,10:26:21,61,0.983606557
+2105,1,20130417,1,10:26:22,60,1
+2106,1,20130417,1,10:26:23,60,1
+2107,1,20130417,1,10:26:24,62,0.967741935
+2108,1,20130417,1,10:26:25,73,0.821917808
+2109,1,20130417,1,10:26:26,68,0.882352941
+2110,1,20130417,1,10:26:27,64,0.9375
+2111,1,20130417,1,10:26:28,67,0.895522388
+2112,1,20130417,1,10:26:28,72,0.833333333
+2113,1,20130417,1,10:26:29,66,0.909090909
+2114,1,20130417,1,10:26:30,63,0.952380952
+2115,1,20130417,1,10:26:31,62,0.967741935
+2116,1,20130417,1,10:26:32,65,0.923076923
+2117,1,20130417,1,10:26:33,67,0.895522388
+2118,1,20130417,1,10:26:34,65,0.923076923
+2119,1,20130417,1,10:26:35,67,0.895522388
+2120,1,20130417,1,10:26:36,70,0.857142857
+2121,1,20130417,1,10:26:37,64,0.9375
+2122,1,20130417,1,10:26:38,60,1
+2123,1,20130417,1,10:26:39,63,0.952380952
+2124,1,20130417,1,10:26:39,68,0.882352941
+2125,1,20130417,1,10:26:40,66,0.909090909
+2126,1,20130417,1,10:26:41,64,0.9375
+2127,1,20130417,1,10:26:42,66,0.909090909
+2128,1,20130417,1,10:26:43,74,0.810810811
+2129,1,20130417,1,10:26:44,69,0.869565217
+2130,1,20130417,1,10:26:45,64,0.9375
+2131,1,20130417,1,10:26:46,65,0.923076923
+2132,1,20130417,1,10:26:47,72,0.833333333
+2133,1,20130417,1,10:26:48,65,0.923076923
+2134,1,20130417,1,10:26:48,70,0.857142857
+2135,1,20130417,1,10:26:49,78,0.769230769
+2136,1,20130417,1,10:26:50,70,0.857142857
+2137,1,20130417,1,10:26:51,58,1.034482759
+2138,1,20130417,1,10:26:52,61,0.983
<TRUNCATED>
[7/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
[ASTERIXDB-2083][COMP][RT][IDX][SITE] Budget-Constrained Inverted index search
- user-model changes: add text.searchmemory parameter
- storage format changes: no
- interface changes: IInvertedIndexSearcher, IInPlaceInvertedIndex,
IInvertedIndexAccessor, IInvertedListCursor
IObjectFactory, IPartitionedInvertedIndex,
IIndexAccessor
Details:
- Introduce text.searchmemory parameter in the configuration
to conduct budget-constrained inverted index search to prevent
a possible OOM exception
- Remove non-standard hyracks task context from the inverted-index-search
Change-Id: Ib2b2ef7c0b8c55ef66a5322be5d97ebbbf287bf5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2251
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/afe0d3d9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/afe0d3d9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/afe0d3d9
Branch: refs/heads/master
Commit: afe0d3d99d42260a0a8553495354720ee8d62a84
Parents: c587da1
Author: Taewoo Kim <wa...@yahoo.com>
Authored: Sun Feb 18 19:12:08 2018 -0800
Committer: Taewoo Kim <wa...@gmail.com>
Committed: Mon Feb 19 09:07:23 2018 -0800
----------------------------------------------------------------------
.../physical/InvertedIndexPOperator.java | 10 +-
.../asterix-app/data/csv/fragile_sample.csv | 20000 +++++++++++++++++
.../apache/asterix/api/common/APIFramework.java | 18 +-
.../app/resource/OperatorResourcesComputer.java | 26 +-
.../org/apache/asterix/utils/ResourceUtils.java | 7 +-
.../asterix-app/src/main/resources/cc.conf | 1 +
.../asterix-app/src/main/resources/cc2.conf | 1 +
.../asterix-app/src/main/resources/cc3.conf | 1 +
.../app/resource/PlanStagesGeneratorTest.java | 2 +-
.../asterix-app/src/test/resources/cc.conf | 1 +
.../fulltext-index-03.1.ddl.sqlpp | 51 +
.../fulltext-index-03.2.update.sqlpp | 24 +
.../fulltext-index-03.3.query.sqlpp | 25 +
.../fulltext-index-large-data.1.ddl.sqlpp | 47 +
.../fulltext-index-large-data.2.update.sqlpp | 27 +
.../fulltext-index-large-data.3.ddl.sqlpp | 22 +
.../fulltext-index-large-data.4.query.sqlpp | 24 +
.../fulltext-index-large-data.5.query.sqlpp | 24 +
...x-ngram-edit-distance-large-data.1.ddl.sqlpp | 47 +
...gram-edit-distance-large-data.2.update.sqlpp | 27 +
...x-ngram-edit-distance-large-data.3.ddl.sqlpp | 22 +
...ngram-edit-distance-large-data.4.query.sqlpp | 26 +
...ngram-edit-distance-large-data.5.query.sqlpp | 26 +
.../cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less.1.regexadm | 1 +
.../fulltext-index-large-data.4.adm | 1 +
.../fulltext-index-large-data.5.adm | 1 +
...d-index-ngram-edit-distance-large-data.4.adm | 1 +
...d-index-ngram-edit-distance-large-data.5.adm | 1 +
.../resources/runtimets/testsuite_sqlpp.xml | 15 +
.../common/config/CompilerProperties.java | 10 +
.../asterix-doc/src/site/markdown/ncservice.md | 1 +
.../base/PhysicalOptimizationConfig.java | 10 +
.../hyracks/api/exceptions/ErrorCode.java | 5 +
.../hyracks/api/util/HyracksConstants.java | 4 +
.../src/main/resources/errormsg/en.properties | 5 +
.../BufferManagerBackedVSizeFrame.java | 108 +
.../BTreeSearchOperatorNodePushable.java | 7 +
.../IndexSearchOperatorNodePushable.java | 6 +-
.../common/impls/NoOpIndexAccessParameters.java | 5 +-
.../am/common/test/IIndexCursorTest.java | 2 +-
...eeDiskComponentScanOperatorNodePushable.java | 6 +
.../lsm/btree/impls/ExternalBTreeWithBuddy.java | 4 +-
.../storage/am/lsm/btree/impls/LSMBTree.java | 11 +-
.../am/lsm/common/impls/AbstractLSMIndex.java | 13 +-
.../api/IInPlaceInvertedIndex.java | 19 +-
.../api/IInvertedIndexAccessor.java | 6 +-
.../api/IInvertedIndexSearcher.java | 33 +-
.../invertedindex/api/IInvertedListCursor.java | 57 -
.../lsm/invertedindex/api/IObjectFactory.java | 4 +-
.../api/IPartitionedInvertedIndex.java | 6 +-
.../invertedindex/api/InvertedListCursor.java | 103 +
...SMInvertedIndexSearchOperatorDescriptor.java | 8 +-
...InvertedIndexSearchOperatorNodePushable.java | 24 +-
.../invertedindex/impls/LSMInvertedIndex.java | 19 +-
.../impls/LSMInvertedIndexAccessor.java | 6 +-
.../impls/LSMInvertedIndexOpContext.java | 21 +-
...SMInvertedIndexSearchCursorInitialState.java | 47 +-
.../inmemory/InMemoryInvertedIndex.java | 27 +-
.../inmemory/InMemoryInvertedIndexAccessor.java | 46 +-
.../inmemory/InMemoryInvertedListCursor.java | 54 +-
.../PartitionedInMemoryInvertedIndex.java | 15 +-
...artitionedInMemoryInvertedIndexAccessor.java | 16 +-
.../FixedSizeElementInvertedListCursor.java | 394 +-
.../FixedSizeElementInvertedListScanCursor.java | 191 +
.../ondisk/FixedSizeFrameTupleAccessor.java | 11 +-
.../ondisk/FixedSizeFrameTupleAppender.java | 32 +-
.../ondisk/OnDiskInvertedIndex.java | 128 +-
.../ondisk/OnDiskInvertedIndexOpContext.java | 2 +-
.../OnDiskInvertedIndexRangeSearchCursor.java | 78 +-
.../ondisk/OnDiskInvertedIndexSearchCursor.java | 73 +-
.../ondisk/PartitionedOnDiskInvertedIndex.java | 42 +-
.../search/AbstractTOccurrenceSearcher.java | 185 +-
.../search/InvertedIndexFinalSearchResult.java | 179 +
.../search/InvertedIndexSearchResult.java | 416 +
.../search/InvertedListCursorFactory.java | 14 +-
.../search/InvertedListMerger.java | 661 +-
.../search/InvertedListPartitions.java | 27 +-
.../search/PartitionedTOccurrenceSearcher.java | 236 +-
.../lsm/invertedindex/search/SearchResult.java | 146 -
.../search/TOccurrenceSearcher.java | 72 +-
.../am/lsm/invertedindex/util/ObjectCache.java | 7 +-
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 10 +-
.../storage/am/lsm/rtree/impls/LSMRTree.java | 3 +-
.../impls/LSMRTreeWithAntiMatterTuples.java | 2 +-
.../RTreeSearchOperatorNodePushable.java | 7 +
.../hyracks/storage/common/IIndexAccessor.java | 4 +-
.../am/common/AbstractIndexTestWorker.java | 2 +-
.../storage/am/common/IndexTestContext.java | 2 +-
.../am/config/AccessMethodTestsConfig.java | 3 +
.../cursor/LSMBTreePointSearchCursorTest.java | 2 +-
.../cursor/LSMBTreeRangeSearchCursorTest.java | 2 +-
.../pom.xml | 5 +
.../common/AbstractInvertedIndexDeleteTest.java | 2 +-
.../common/AbstractInvertedIndexSearchTest.java | 2 +-
.../common/AbstractInvertedIndexTest.java | 2 +-
.../multithread/LSMInvertedIndexTestWorker.java | 24 +
.../ondisk/FixedSizeFrameTupleTest.java | 12 +-
.../util/LSMInvertedIndexTestContext.java | 24 +
.../util/LSMInvertedIndexTestUtils.java | 176 +-
101 files changed, 23388 insertions(+), 979 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index c3cc0ae..eeb6688 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.algebra.operators.physical;
+import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -109,13 +110,16 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
// tuples.
retainNull = true;
}
+ // In-memory budget (frame limit) for inverted-index search operations
+ int frameLimit = OptimizationConfUtil.getPhysicalOptimizationConfig().getMaxFramesForTextSearch();
+
// Build runtime.
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch =
buildInvertedIndexRuntime(metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema,
jobGenParams.getRetainInput(), retainNull, jobGenParams.getDatasetName(), dataset,
jobGenParams.getIndexName(), jobGenParams.getSearchKeyType(), keyIndexes,
jobGenParams.getSearchModifierType(), jobGenParams.getSimilarityThreshold(),
- minFilterFieldIndexes, maxFilterFieldIndexes, jobGenParams.getIsFullTextSearch());
+ minFilterFieldIndexes, maxFilterFieldIndexes, jobGenParams.getIsFullTextSearch(), frameLimit);
// Contribute operator in hyracks job.
builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first);
@@ -129,7 +133,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
AbstractUnnestMapOperator unnestMap, IOperatorSchema opSchema, boolean retainInput, boolean retainMissing,
String datasetName, Dataset dataset, String indexName, ATypeTag searchKeyType, int[] keyFields,
SearchModifierType searchModifierType, IAlgebricksConstantValue similarityThreshold,
- int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery)
+ int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery, int frameLimit)
throws AlgebricksException {
boolean propagateIndexFilter = unnestMap.propagateIndexFilter();
IAObject simThresh = ((AsterixConstantValue) similarityThreshold).getObject();
@@ -159,7 +163,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(), secondaryIndex,
IndexOperation.SEARCH, null),
minFilterFieldIndexes, maxFilterFieldIndexes, isFullTextSearchQuery, numPrimaryKeys,
- propagateIndexFilter);
+ propagateIndexFilter, frameLimit);
return new Pair<>(invIndexSearchOp, secondarySplitsAndConstraint.second);
}
}
[5/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index c15704a..ad715a4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -118,17 +118,20 @@ public class APIFramework {
private static final int MIN_FRAME_LIMIT_FOR_SORT = 3;
private static final int MIN_FRAME_LIMIT_FOR_GROUP_BY = 4;
private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5;
+ // one for query, two for intermediate results, one for final result, and one for reading an inverted list
+ private static final int MIN_FRAME_LIMIT_FOR_TEXTSEARCH = 5;
private static final String LPLAN = "Logical plan";
private static final String OPLAN = "Optimized logical plan";
// A white list of supported configurable parameters.
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
- CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
- FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME,
- FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION,
- FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, "inline_with",
- "hash_merge", "output-record-type", AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION);
+ CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
+ CompilerProperties.COMPILER_PARALLELISM_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
+ StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
+ FeedActivityDetails.COLLECT_LOCATIONS, "inline_with", "hash_merge", "output-record-type",
+ AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
@@ -341,11 +344,16 @@ public class APIFramework {
int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY,
querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY),
compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN);
+ int textSearchFrameLimit = getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
+ querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY),
+ compilerProperties.getTextSearchMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_TEXTSEARCH);
final PhysicalOptimizationConfig physOptConf = OptimizationConfUtil.getPhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
physOptConf.setMaxFramesExternalGroupBy(groupFrameLimit);
physOptConf.setMaxFramesForJoin(joinFrameLimit);
+ physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
+
return physOptConf;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
index 7eaaa3d..1913683 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
@@ -22,6 +22,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
public class OperatorResourcesComputer {
@@ -33,14 +34,16 @@ public class OperatorResourcesComputer {
private final long groupByMemorySize;
private final long joinMemorySize;
private final long sortMemorySize;
+ private final long textSearchMemorySize;
private final long frameSize;
public OperatorResourcesComputer(int numComputationPartitions, int sortFrameLimit, int groupFrameLimit,
- int joinFrameLimit, long frameSize) {
+ int joinFrameLimit, int textSearchFrameLimit, long frameSize) {
this.numComputationPartitions = numComputationPartitions;
this.groupByMemorySize = groupFrameLimit * frameSize;
this.joinMemorySize = joinFrameLimit * frameSize;
this.sortMemorySize = sortFrameLimit * frameSize;
+ this.textSearchMemorySize = textSearchFrameLimit * frameSize;
this.frameSize = frameSize;
}
@@ -62,7 +65,6 @@ public class OperatorResourcesComputer {
case EMPTYTUPLESOURCE:
case DELEGATE_OPERATOR:
case EXTERNAL_LOOKUP:
- case LEFT_OUTER_UNNEST_MAP:
case LIMIT:
case MATERIALIZE:
case NESTEDTUPLESOURCE:
@@ -78,7 +80,6 @@ public class OperatorResourcesComputer {
case UNIONALL:
case UNNEST:
case LEFT_OUTER_UNNEST:
- case UNNEST_MAP:
case UPDATE:
case WRITE:
case WRITE_RESULT:
@@ -86,6 +87,15 @@ public class OperatorResourcesComputer {
case INSERT_DELETE_UPSERT:
case INTERSECT:
return getOperatorRequiredMemory(operator, frameSize);
+ case LEFT_OUTER_UNNEST_MAP:
+ case UNNEST_MAP:
+ // Since an inverted-index search requires certain amount of memory, needs to calculate
+ // the memory size differently if the given index-search is an inverted-index search.
+ long unnestMapMemorySize = frameSize;
+ if (isInvertedIndexSearch((AbstractUnnestMapOperator) operator)) {
+ unnestMapMemorySize += textSearchMemorySize;
+ }
+ return getOperatorRequiredMemory(operator, unnestMapMemorySize);
case EXCHANGE:
return getExchangeRequiredMemory((ExchangeOperator) operator);
case GROUP:
@@ -108,6 +118,16 @@ public class OperatorResourcesComputer {
return memorySize;
}
+ private boolean isInvertedIndexSearch(AbstractUnnestMapOperator op) {
+ IPhysicalOperator physicalOperator = op.getPhysicalOperator();
+ final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag();
+ if (physicalOperatorTag == PhysicalOperatorTag.LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH
+ || physicalOperatorTag == PhysicalOperatorTag.SINGLE_PARTITION_INVERTED_INDEX_SEARCH) {
+ return true;
+ }
+ return false;
+ }
+
private long getExchangeRequiredMemory(ExchangeOperator op) {
final IPhysicalOperator physicalOperator = op.getPhysicalOperator();
final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
index 0149ffa..d9ead33 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
@@ -58,9 +58,10 @@ public class ResourceUtils {
final int sortFrameLimit = physicalOptimizationConfig.getMaxFramesExternalSort();
final int groupFrameLimit = physicalOptimizationConfig.getMaxFramesForGroupBy();
final int joinFrameLimit = physicalOptimizationConfig.getMaxFramesForJoin();
+ final int textSearchFrameLimit = physicalOptimizationConfig.getMaxFramesForTextSearch();
final List<PlanStage> planStages = getStages(plan);
return getStageBasedRequiredCapacity(planStages, computationLocations.getLocations().length, sortFrameLimit,
- groupFrameLimit, joinFrameLimit, frameSize);
+ groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize);
}
public static List<PlanStage> getStages(ILogicalPlan plan) throws AlgebricksException {
@@ -72,9 +73,9 @@ public class ResourceUtils {
}
public static IClusterCapacity getStageBasedRequiredCapacity(List<PlanStage> stages, int computationLocations,
- int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int frameSize) {
+ int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int textSearchFrameLimit, int frameSize) {
final OperatorResourcesComputer computer = new OperatorResourcesComputer(computationLocations, sortFrameLimit,
- groupFrameLimit, joinFrameLimit, frameSize);
+ groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize);
final IClusterCapacity clusterCapacity = new ClusterCapacity();
final Long maxRequiredMemory = stages.stream().mapToLong(stage -> stage.getRequiredMemory(computer)).max()
.orElseThrow(IllegalStateException::new);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index fe6cf64..0d9f54f 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc2.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 113497d..ddf1438 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
compiler.parallelism=-1
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc3.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index e210bf62..b819f24 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
compiler.parallelism=3
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
index 0e55b1e..d3113ca 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
@@ -301,7 +301,7 @@ public class PlanStagesGeneratorTest {
private void assertRequiredMemory(List<PlanStage> stages, long expectedMemory) {
final IClusterCapacity clusterCapacity = ResourceUtils.getStageBasedRequiredCapacity(stages, PARALLELISM,
- FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_SIZE);
+ FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_SIZE);
Assert.assertEquals(clusterCapacity.getAggregatedMemoryByteSize(), expectedMemory);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 5cf1bbe..1c141f1 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -52,5 +52,6 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
new file mode 100644
index 0000000..22ad574
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Full-text search index test
+ * : This test is intended to verify that the full-text search works as expected.
+ * : query #3 - two string values in [an ordered list] query with "any" option
+ * : in this case, "any" option that enforces a disjunctive search will be applied.
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index fulltext_index_title on MyData(title) type fulltext;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
new file mode 100644
index 0000000..9b7c1f6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset MyData
+using localfs
+(("path"="asterix_nc1://data/spatial/spatialData2.json"),("format"="adm"));
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
new file mode 100644
index 0000000..28c4190
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select element {"id":ftval.id}
+from MyData as ftval
+where test.ftcontains(ftval.title, ["object","database"], {"mode":"any"})
+order by ftval.id;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
new file mode 100644
index 0000000..e63ecdb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Testing using a secondary fulltext index
+ * : Based on the text.searchmemory budget (default=160KB), this test should
+ * : create intermediate result files.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+// For bulk-loading test (tuples will be fetched from the disk)
+create dataset Fragile_raw (FragileTypeRaw) primary key row_id;
+
+// For insert test (tuples will be fetched from memory)
+create dataset Fragile_raw_insert (FragileTypeRaw) primary key row_id;
+create index fulltext_insert_index on Fragile_raw_insert(date) type fulltext;
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
new file mode 100644
index 0000000..3333df8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Fragile_raw using localfs
+((`path`=`asterix_nc1://data/csv/fragile_sample.csv`),(`format`=`delimited-text`),(`delimiter`=`,`)) pre-sorted;
+
+insert into Fragile_raw_insert (
+ select value t from Fragile_raw t
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
new file mode 100644
index 0000000..5d719c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on Fragile_raw(date) type ngram(2);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
new file mode 100644
index 0000000..a41a95f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value count(*)
+from Fragile_raw as o
+where ftcontains(o.date,'20130417');
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
new file mode 100644
index 0000000..81dd172
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value count(*)
+from Fragile_raw_insert as o
+where ftcontains(o.date,'20130417');
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
new file mode 100644
index 0000000..b7f0ec5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Testing using a secondary inverted n-gram index
+ * : Based on the text.searchmemory budget (default=160KB), this test should
+ * : create intermediate result files.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+// For bulk-loading test (tuples will be fetched from the disk)
+create dataset Fragile_raw (FragileTypeRaw) primary key row_id;
+
+// For insert test (tuples will be fetched from memory)
+create dataset Fragile_raw_insert (FragileTypeRaw) primary key row_id;
+create index ngram_insert_index on Fragile_raw_insert(date) type ngram(2);
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
new file mode 100644
index 0000000..3333df8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Fragile_raw using localfs
+((`path`=`asterix_nc1://data/csv/fragile_sample.csv`),(`format`=`delimited-text`),(`delimiter`=`,`)) pre-sorted;
+
+insert into Fragile_raw_insert (
+ select value t from Fragile_raw t
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
new file mode 100644
index 0000000..5d719c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on Fragile_raw(date) type ngram(2);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
new file mode 100644
index 0000000..64a71ce
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The following predicate returns all rows since there are only two distinct date values on the dataset.
+
+use test;
+
+select value count(*)
+from Fragile_raw as o
+where `edit-distance-check`(o.date,'20130417',4)[0];
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
new file mode 100644
index 0000000..c68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The following predicate returns all rows since there are only two distinct date values on the dataset.
+
+use test;
+
+select value count(*)
+from Fragile_raw_insert as o
+where `edit-distance-check`(o.date,'20130417',4)[0];
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 59057b1..28ef0eb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : 0,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "INFO",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 13dacb8..61cf528 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : -1,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "WARN",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9f18638..d340386 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : 3,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "WARN",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
new file mode 100644
index 0000000..1746da6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
@@ -0,0 +1 @@
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
new file mode 100644
index 0000000..1746da6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
@@ -0,0 +1 @@
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
@@ -0,0 +1 @@
+20000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
@@ -0,0 +1 @@
+20000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 961fb41..909b523 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -2744,10 +2744,20 @@
</compilation-unit>
</test-case>
<test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-03">
+ <output-dir compare="Text">fulltext-index-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
<compilation-unit name="fulltext-index-08">
<output-dir compare="Text">fulltext-index-08</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-large-data">
+ <output-dir compare="Text">fulltext-index-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="global-aggregate">
<test-case FilePath="global-aggregate">
@@ -3169,6 +3179,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-large-data">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
<compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
<output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
</compilation-unit>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 0580c1f..6d89ff5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -42,6 +42,10 @@ public class CompilerProperties extends AbstractProperties {
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
"The memory budget (in bytes) for a group by operator instance in a partition"),
+ COMPILER_TEXTSEARCHMEMORY(
+ LONG_BYTE_UNIT,
+ StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
+ "The memory budget (in bytes) for an inverted-index-search operator instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@ -98,6 +102,8 @@ public class CompilerProperties extends AbstractProperties {
public static final String COMPILER_JOINMEMORY_KEY = Option.COMPILER_JOINMEMORY.ini();
+ public static final String COMPILER_TEXTSEARCHMEMORY_KEY = Option.COMPILER_TEXTSEARCHMEMORY.ini();
+
public static final String COMPILER_PARALLELISM_KEY = Option.COMPILER_PARALLELISM.ini();
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
@@ -118,6 +124,10 @@ public class CompilerProperties extends AbstractProperties {
return accessor.getLong(Option.COMPILER_GROUPMEMORY);
}
+ public long getTextSearchMemorySize() {
+ return accessor.getLong(Option.COMPILER_TEXTSEARCHMEMORY);
+ }
+
public int getFrameSize() {
return accessor.getInt(Option.COMPILER_FRAMESIZE);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-doc/src/site/markdown/ncservice.md
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-doc/src/site/markdown/ncservice.md b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
index aadecc2..1d174e9 100644
--- a/asterixdb/asterix-doc/src/site/markdown/ncservice.md
+++ b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
@@ -345,6 +345,7 @@ The following parameters are configured under the "[common]" section.
| common | compiler.joinmemory | The memory budget (in bytes) for a join operator instance in a partition | 33554432 (32 MB) |
| common | compiler.parallelism | The degree of parallelism for query execution. Zero means to use the storage parallelism as the query execution parallelism, while other integer values dictate the number of query execution parallel partitions. The system will fall back to use the number of all available CPU cores in the cluster as the degree of parallelism if the number set by a user is too large or too small | 0 |
| common | compiler.sortmemory | The memory budget (in bytes) for a sort operator instance in a partition | 33554432 (32 MB) |
+| common | compiler.textsearchmemory | The memory budget (in bytes) for an inverted-index-search operator instance in a partition | 33554432 (32 MB) |
| common | instance.name | The name of this cluster instance | DEFAULT_INSTANCE |
| common | log.level | The logging level for master and slave processes | WARNING |
| common | max.wait.active.cluster | The max pending time (in seconds) for cluster startup. After the threshold, if the cluster still is not up and running, it is considered unavailable | 60 |
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index a242927..07ff0ab 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -28,6 +28,7 @@ public class PhysicalOptimizationConfig {
private static final String MAX_FRAMES_EXTERNAL_GROUP_BY = "MAX_FRAMES_EXTERNAL_GROUP_BY";
private static final String MAX_FRAMES_FOR_JOIN_LEFT_INPUT = "MAX_FRAMES_FOR_JOIN_LEFT_INPUT";
private static final String MAX_FRAMES_FOR_JOIN = "MAX_FRAMES_FOR_JOIN";
+ private static final String MAX_FRAMES_FOR_TEXTSEARCH = "MAX_FRAMES_FOR_TEXTSEARCH";
private static final String FUDGE_FACTOR = "FUDGE_FACTOR";
private static final String MAX_RECORDS_PER_FRAME = "MAX_RECORDS_PER_FRAME";
@@ -109,6 +110,15 @@ public class PhysicalOptimizationConfig {
setInt(MAX_FRAMES_EXTERNAL_SORT, frameLimit);
}
+ public int getMaxFramesForTextSearch() {
+ int frameSize = getFrameSize();
+ return getInt(MAX_FRAMES_FOR_TEXTSEARCH, (int) (((long) 32 * MB) / frameSize));
+ }
+
+ public void setMaxFramesForTextSearch(int frameLimit) {
+ setInt(MAX_FRAMES_FOR_TEXTSEARCH, frameLimit);
+ }
+
public int getHashGroupByTableSize() {
return getInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index ad2d6f9..0dca782 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -142,6 +142,11 @@ public class ErrorCode {
public static final int CANNOT_DEACTIVATE_PINNED_BLOOM_FILTER = 106;
public static final int PREDICATE_CANNOT_BE_NULL = 107;
public static final int FULLTEXT_ONLY_EXECUTABLE_FOR_STRING_OR_LIST = 108;
+ public static final int NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH = 109;
+ public static final int CANNOT_CONTINUE_TEXT_SEARCH_HYRACKS_TASK_IS_NULL = 110;
+ public static final int CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL = 111;
+ public static final int CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT = 112;
+ public static final int UNDEFINED_INVERTED_LIST_MERGE_TYPE = 113;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
index 5609721..9314f24 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
@@ -21,6 +21,10 @@ package org.apache.hyracks.api.util;
public class HyracksConstants {
public static final String KEY_MESSAGE = "HYX:MSG";
public static final String HYRACKS_LOGGER_NAME = "org.apache.hyracks";
+ // A frame manager that manages all inverted index searches
+ public static final String INVERTED_INDEX_SEARCH_FRAME_MANAGER = "INVERTED_INDEX_SEARCH_FRAME_MANAGER";
+ // Hyracks task context
+ public static final String HYRACKS_TASK_CONTEXT = "HYRACKS_TASK_CONTEXT";
private HyracksConstants() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index c3945a3..50e1ad4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -125,5 +125,10 @@
106 = Failed to deactivate the bloom filter since it is pinned by other users
107 = The given search predicate can't be null.
108 = Full-text search can be only executed on STRING or (UN)ORDERED LIST.
+109 = Not enough memory is allocated to %1$s. Please assign more memory to text search.
+110 = To conduct an inverted-index search, the given Hyracks task context cannot be null.
+111 = To conduct an inverted-index search, the given buffer manager cannot be null.
+112 = Cannot add an element to an inverted-index search result.
+113 = Undefined inverted-list merge type: %1$s
10000 = The given rule collection %1$s is not an instance of the List class.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
new file mode 100644
index 0000000..bf5e803
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.context.IHyracksFrameMgrContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Variable size frame managed by a buffer manager. The frame size can be changed.
+ * The caller of this class needs to ensure that this class releases its buffer by calling close() method.
+ */
+public class BufferManagerBackedVSizeFrame implements IFrame {
+
+ protected final int minFrameSize;
+ protected final IHyracksFrameMgrContext ctx;
+ protected final ISimpleFrameBufferManager bufferManager;
+ protected ByteBuffer frame;
+
+ public BufferManagerBackedVSizeFrame(IHyracksFrameMgrContext ctx, ISimpleFrameBufferManager bufferManager)
+ throws HyracksDataException {
+ this(ctx, ctx.getInitialFrameSize(), bufferManager);
+ }
+
+ public BufferManagerBackedVSizeFrame(IHyracksFrameMgrContext ctx, int frameSize,
+ ISimpleFrameBufferManager bufferManager) throws HyracksDataException {
+ this.ctx = ctx;
+ this.minFrameSize = ctx.getInitialFrameSize();
+ this.bufferManager = bufferManager;
+ frame = bufferManager.acquireFrame(frameSize);
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return frame;
+ }
+
+ @Override
+ public void ensureFrameSize(int newSize) throws HyracksDataException {
+ if (newSize > getFrameSize()) {
+ bufferManager.releaseFrame(frame);
+ frame = bufferManager.acquireFrame(newSize);
+ }
+ }
+
+ @Override
+ public void resize(int frameSize) throws HyracksDataException {
+ if (getFrameSize() != frameSize) {
+ bufferManager.releaseFrame(frame);
+ frame = bufferManager.acquireFrame(frameSize);
+ }
+ }
+
+ @Override
+ public int getFrameSize() {
+ return frame.capacity();
+ }
+
+ @Override
+ public int getMinSize() {
+ return minFrameSize;
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ resize(minFrameSize);
+ Arrays.fill(frame.array(), (byte) 0);
+ frame.clear();
+ }
+
+ /**
+ * Allocates an initial frame and clears it.
+ */
+ public void acquireFrame() throws HyracksDataException {
+ if (frame == null) {
+ frame = bufferManager.acquireFrame(minFrameSize);
+ }
+ reset();
+ }
+
+ public void destroy() throws HyracksDataException {
+ if (frame != null) {
+ bufferManager.releaseFrame(frame);
+ frame = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 72941f3..6bbf437 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -104,4 +105,10 @@ public class BTreeSearchOperatorNodePushable extends IndexSearchOperatorNodePush
protected int getFieldCount() {
return ((ITreeIndex) index).getFieldCount();
}
+
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ // No additional parameters are required for the B+Tree search case
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index ab9359f..eb493af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -144,7 +144,10 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
protected abstract void resetSearchPredicate(int tupleIndex);
- protected IIndexCursor createCursor() {
+ // Assigns any index-type specific related accessor parameters
+ protected abstract void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException;
+
+ protected IIndexCursor createCursor() throws HyracksDataException {
return indexAccessor.createSearchCursor(false);
}
@@ -184,6 +187,7 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
ISearchOperationCallback searchCallback =
searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
IIndexAccessParameters iap = new IndexAccessParameters(NoOpOperationCallback.INSTANCE, searchCallback);
+ addAdditionalIndexAccessorParams(iap);
indexAccessor = index.createAccessor(iap);
cursor = createCursor();
if (retainInput) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
index 3618cae..2edea70 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.common.impls;
+import java.util.Collections;
import java.util.Map;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
@@ -26,6 +27,8 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
public class NoOpIndexAccessParameters implements IIndexAccessParameters {
public static final NoOpIndexAccessParameters INSTANCE = new NoOpIndexAccessParameters();
+ // Immutable empty map not to cause getParameters().get() generate an exception
+ private static final Map<String, Object> paramMap = Collections.emptyMap();
private NoOpIndexAccessParameters() {
}
@@ -42,7 +45,7 @@ public class NoOpIndexAccessParameters implements IIndexAccessParameters {
@Override
public Map<String, Object> getParameters() {
- return null;
+ return paramMap;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
index 6a3fdf1..418b370 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
@@ -279,7 +279,7 @@ public abstract class IIndexCursorTest {
Assert.assertNull(cursor.getTuple());
}
- protected IIndexCursor createCursor(IIndexAccessor accessor) {
+ protected IIndexCursor createCursor(IIndexAccessor accessor) throws Exception {
return accessor.createSearchCursor(false);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
index 1f71876..860d36e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
public class LSMBTreeDiskComponentScanOperatorNodePushable extends IndexSearchOperatorNodePushable {
@@ -66,4 +67,9 @@ public class LSMBTreeDiskComponentScanOperatorNodePushable extends IndexSearchOp
return ((ITreeIndex) index).getFieldCount() + 2;
}
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ // no additional parameters are required for the B+Tree search case
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 0be5556..6e06d37 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -61,7 +61,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -633,8 +632,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- protected AbstractLSMIndexOperationContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ protected AbstractLSMIndexOperationContext createOpContext(IIndexAccessParameters iap) {
return null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 4610fd2..4578eb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -65,8 +65,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICurs
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -394,18 +392,17 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
@Override
- public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ public LSMBTreeOpContext createOpContext(IIndexAccessParameters iap) {
int numBloomFilterKeyFields = hasBloomFilter
? ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length : 0;
return new LSMBTreeOpContext(this, memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
- modificationCallback, searchCallback, numBloomFilterKeyFields, getTreeFields(), getFilterFields(),
- getHarness(), getFilterCmpFactories(), tracer);
+ iap.getModificationCallback(), iap.getSearchOperationCallback(), numBloomFilterKeyFields,
+ getTreeFields(), getFilterFields(), getHarness(), getFilterCmpFactories(), tracer);
}
@Override
public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) {
- return createAccessor(createOpContext(iap.getModificationCallback(), iap.getSearchOperationCallback()));
+ return createAccessor(createOpContext(iap));
}
public ILSMIndexAccessor createAccessor(AbstractLSMIndexOperationContext opCtx) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 908a5c2..fef5516 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -38,7 +38,6 @@ import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
-import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -63,10 +62,9 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.logging.log4j.Level;
@@ -356,8 +354,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- AbstractLSMIndexOperationContext opCtx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(ctx.getComponentHolder());
ILSMIOOperation flushOp = createFlushOperation(opCtx, componentFileRefs, callback);
@@ -369,8 +366,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
throws HyracksDataException {
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
// merge must create a different op ctx
- AbstractLSMIndexOperationContext opCtx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(mergingComponents);
ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
@@ -762,8 +758,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
protected abstract LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
ILSMDiskComponent lastComponent) throws HyracksDataException;
- protected abstract AbstractLSMIndexOperationContext createOpContext(
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback)
+ protected abstract AbstractLSMIndexOperationContext createOpContext(IIndexAccessParameters iap)
throws HyracksDataException;
protected abstract ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
index 6f583b4..27966d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
@@ -18,18 +18,29 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
public interface IInPlaceInvertedIndex extends IInvertedIndex {
/**
- * Create an inverted list cursor
+ * Creates an inverted list cursor.
+ *
+ * @throws HyracksDataException
+ */
+ InvertedListCursor createInvertedListCursor(IHyracksTaskContext ctx) throws HyracksDataException;
+
+ /**
+ * Creates an inverted-list-range-search cursor. This cursor is mainly used to conduct
+ * a full scan of an inverted list during a storage-component-merge operation.
+ *
+ * @throws HyracksDataException
*/
- IInvertedListCursor createInvertedListCursor();
+ InvertedListCursor createInvertedListRangeSearchCursor() throws HyracksDataException;
/**
- * Open an inverted list cursor
+ * Opens an inverted list cursor
*
* @param listCursor
* the cursor to open
@@ -39,6 +50,6 @@ public interface IInPlaceInvertedIndex extends IInvertedIndex {
* the operation context under which the cursor is to be open
* @throws HyracksDataException
*/
- void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
+ void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
index 86c0bf1..fe29c5f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
@@ -26,12 +26,12 @@ import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
public interface IInvertedIndexAccessor extends IIndexAccessor {
- public IInvertedListCursor createInvertedListCursor();
+ public InvertedListCursor createInvertedListCursor() throws HyracksDataException;
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException;
- public IIndexCursor createRangeSearchCursor();
+ public IIndexCursor createRangeSearchCursor() throws HyracksDataException;
public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
index 17e8ad2..837870f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
@@ -19,27 +19,36 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
+import org.apache.hyracks.storage.common.IIndexCursor;
+/**
+ * Inverted index search cursor class that conducts the actual inverted index traversal
+ *
+ */
public interface IInvertedIndexSearcher {
- public void search(OnDiskInvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred,
- IIndexOperationContext ictx) throws HyracksDataException;
+ /**
+ * Searches the given inverted index using the search predicate and initializes the result cursor.
+ */
+ public void search(IIndexCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOperationContext ictx)
+ throws HyracksDataException;
- public IFrameTupleAccessor createResultFrameTupleAccessor();
+ /**
+ * Continues the search process if it is paused. (e.g., output buffer full)
+ *
+ * @return true only if all search process is done.
+ * false otherwise.
+ */
+ public boolean continueSearch() throws HyracksDataException;
- public ITupleReference createResultFrameTupleReference();
+ public boolean hasNext() throws HyracksDataException;
- public List<ByteBuffer> getResultBuffers();
+ public void next() throws HyracksDataException;
- public int getNumValidResultBuffers();
+ public void destroy() throws HyracksDataException;
- public void reset();
+ public ITupleReference getTuple();
}
[3/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java
index 79033d2..23854f9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java
@@ -25,6 +25,11 @@ import org.apache.hyracks.api.comm.FrameHelper;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+/**
+ * This is a fixed-size tuple accessor class.
+ * The frame structure: [4 bytes for minimum Hyracks frame count] [fixed-size tuple 1] ... [fixed-size tuple n] ...
+ * [4 bytes for the tuple count in a frame]
+ */
public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor {
private final int frameSize;
@@ -82,12 +87,12 @@ public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor {
@Override
public int getFieldStartOffset(int tupleIndex, int fIdx) {
- return tupleIndex * tupleSize + fieldStartOffsets[fIdx];
+ return getTupleStartOffset(tupleIndex) + fieldStartOffsets[fIdx];
}
@Override
public int getTupleCount() {
- return buffer.getInt(FrameHelper.getTupleCountOffset(frameSize));
+ return buffer != null ? buffer.getInt(FrameHelper.getTupleCountOffset(frameSize)) : 0;
}
@Override
@@ -97,7 +102,7 @@ public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor {
@Override
public int getTupleStartOffset(int tupleIndex) {
- return tupleIndex * tupleSize;
+ return FixedSizeFrameTupleAppender.MINFRAME_COUNT_SIZE + tupleIndex * tupleSize;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAppender.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAppender.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAppender.java
index 5dd23c4..85d8576 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAppender.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAppender.java
@@ -20,13 +20,25 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import org.apache.hyracks.api.comm.FrameHelper;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+/**
+ * An appender class for an inverted list. Each frame has two integer values at the beginning and at the end.
+ * The first represents the number of minimum Hyracks frames in a frame. Currently, we use 1 for this value.
+ * The latter represents the number of tuples in a frame. This design is required since we may need to use
+ * RunFileWriter and RunFileReader class during the inverted-index-search operation.
+ */
public class FixedSizeFrameTupleAppender {
- private static final int TUPLE_COUNT_SIZE = 4;
+ // At the end of a frame, an integer value is written to keep the tuple count in this frame.
+ public static final int TUPLE_COUNT_SIZE = 4;
+ // At the beginning of a frame, an integer value is written to keep the number of minimum frames in this frame.
+ // For this class, the frame size is equal to the minimum frame size in Hyracks.
+ public static final int MINFRAME_COUNT_SIZE = 4;
+
private final int frameSize;
private final int tupleSize;
private ByteBuffer buffer;
@@ -42,13 +54,22 @@ public class FixedSizeFrameTupleAppender {
tupleSize = tmp;
}
- public void reset(ByteBuffer buffer, boolean clear) {
+ public void reset(ByteBuffer buffer) {
+ reset(buffer, true, 0, MINFRAME_COUNT_SIZE);
+ }
+
+ public void reset(ByteBuffer buffer, boolean clear, int tupleCount, int tupleDataEndOffset) {
this.buffer = buffer;
if (clear) {
- buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), 0);
- tupleCount = 0;
- tupleDataEndOffset = 0;
+ Arrays.fill(this.buffer.array(), (byte) 0);
+ this.buffer.clear();
+ // the number of minimum frames in a frame - it's one.
+ FrameHelper.serializeFrameSize(this.buffer, 1);
}
+ // tuple count
+ this.buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), tupleCount);
+ this.tupleCount = tupleCount;
+ this.tupleDataEndOffset = tupleDataEndOffset;
}
public boolean append(byte[] bytes, int offset) {
@@ -128,4 +149,5 @@ public class FixedSizeFrameTupleAppender {
public ByteBuffer getBuffer() {
return buffer;
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 8e8cb13..2f4f1d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -21,15 +21,14 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
import java.io.DataOutput;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
@@ -48,7 +47,8 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.TOccurrenceSearcher;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
@@ -70,8 +70,6 @@ import org.apache.hyracks.storage.common.file.BufferedFileHandle;
* cannot exceed the size of a Hyracks frame.
*/
public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
- protected final IHyracksCommonContext ctx = new DefaultHyracksCommonContext();
-
// Schema of BTree tuples, set in constructor.
protected final int invListStartPageIdField;
protected final int invListEndPageIdField;
@@ -185,12 +183,17 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
}
@Override
- public IInvertedListCursor createInvertedListCursor() {
- return new FixedSizeElementInvertedListCursor(bufferCache, fileId, invListTypeTraits);
+ public InvertedListCursor createInvertedListCursor(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new FixedSizeElementInvertedListCursor(bufferCache, fileId, invListTypeTraits, ctx);
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey,
+ public InvertedListCursor createInvertedListRangeSearchCursor() throws HyracksDataException {
+ return new FixedSizeElementInvertedListScanCursor(bufferCache, fileId, invListTypeTraits);
+ }
+
+ @Override
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey,
IIndexOperationContext ictx) throws HyracksDataException {
OnDiskInvertedIndexOpContext ctx = (OnDiskInvertedIndexOpContext) ictx;
ctx.getBtreePred().setLowKeyComparator(ctx.getSearchCmp());
@@ -201,16 +204,19 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
try {
if (ctx.getBtreeCursor().hasNext()) {
ctx.getBtreeCursor().next();
- resetInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor);
+ openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor);
} else {
- listCursor.reset(0, 0, 0, 0);
+ LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ initState.setInvertedListInfo(0, 0, 0, 0);
+ listCursor.open(initState, null);
}
} finally {
ctx.getBtreeCursor().close();
}
}
- public void resetInvertedListCursor(ITupleReference btreeTuple, IInvertedListCursor listCursor) {
+ public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor)
+ throws HyracksDataException {
int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField),
btreeTuple.getFieldStart(invListStartPageIdField));
int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField),
@@ -219,7 +225,9 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
btreeTuple.getFieldStart(invListStartOffField));
int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField),
btreeTuple.getFieldStart(invListNumElementsField));
- listCursor.reset(startPageId, endPageId, startOff, numElements);
+ LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ initState.setInvertedListInfo(startPageId, endPageId, startOff, numElements);
+ listCursor.open(initState, null);
}
public final class OnDiskInvertedIndexBulkLoader implements IIndexBulkLoader {
@@ -416,45 +424,48 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
}
public class OnDiskInvertedIndexAccessor implements IInvertedIndexAccessor {
- private final OnDiskInvertedIndex index;
- private final IInvertedIndexSearcher searcher;
- private final IIndexOperationContext opCtx = new OnDiskInvertedIndexOpContext(btree);
+ protected final OnDiskInvertedIndex index;
+ protected final IIndexOperationContext opCtx;
+ protected final IHyracksTaskContext ctx;
+ protected IInvertedIndexSearcher searcher;
private boolean destroyed = false;
- public OnDiskInvertedIndexAccessor(OnDiskInvertedIndex index) throws HyracksDataException {
- this.index = index;
- this.searcher = new TOccurrenceSearcher(ctx, index);
- }
-
- // Let subclasses initialize.
- protected OnDiskInvertedIndexAccessor(OnDiskInvertedIndex index, IInvertedIndexSearcher searcher) {
+ public OnDiskInvertedIndexAccessor(OnDiskInvertedIndex index, IHyracksTaskContext ctx)
+ throws HyracksDataException {
this.index = index;
- this.searcher = searcher;
+ this.ctx = ctx;
+ this.opCtx = new OnDiskInvertedIndexOpContext(btree);
}
@Override
- public IIndexCursor createSearchCursor(boolean exclusive) {
- return new OnDiskInvertedIndexSearchCursor(searcher, index.getInvListTypeTraits().length);
+ public IIndexCursor createSearchCursor(boolean exclusive) throws HyracksDataException {
+ if (searcher == null) {
+ searcher = new TOccurrenceSearcher(index, ctx);
+ }
+ return new OnDiskInvertedIndexSearchCursor(searcher);
}
@Override
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException {
- searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
+ if (searcher == null) {
+ searcher = new TOccurrenceSearcher(index, ctx);
+ }
+ searcher.search(cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
}
@Override
- public IInvertedListCursor createInvertedListCursor() {
- return index.createInvertedListCursor();
+ public InvertedListCursor createInvertedListCursor() throws HyracksDataException {
+ return index.createInvertedListCursor(ctx);
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException {
index.openInvertedListCursor(listCursor, searchKey, opCtx);
}
@Override
- public IIndexCursor createRangeSearchCursor() {
+ public IIndexCursor createRangeSearchCursor() throws HyracksDataException {
return new OnDiskInvertedIndexRangeSearchCursor(index, opCtx);
}
@@ -496,47 +507,8 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
@Override
public OnDiskInvertedIndexAccessor createAccessor(IIndexAccessParameters iap) throws HyracksDataException {
- return new OnDiskInvertedIndexAccessor(this);
- }
-
- // This is just a dummy hyracks context for allocating frames for temporary
- // results during inverted index searches.
- // TODO: In the future we should use the real HyracksTaskContext to track
- // frame usage.
- public static class DefaultHyracksCommonContext implements IHyracksCommonContext {
- private final int FRAME_SIZE = 32768;
-
- @Override
- public int getInitialFrameSize() {
- return FRAME_SIZE;
- }
-
- @Override
- public IIOManager getIoManager() {
- return null;
- }
-
- @Override
- public ByteBuffer allocateFrame() {
- return ByteBuffer.allocate(FRAME_SIZE);
- }
-
- @Override
- public ByteBuffer allocateFrame(int bytes) throws HyracksDataException {
- return ByteBuffer.allocate(bytes);
- }
-
- @Override
- public ByteBuffer reallocateFrame(ByteBuffer bytes, int newSizeInBytes, boolean copyOldData)
- throws HyracksDataException {
- throw new HyracksDataException("TODO");
- }
-
- @Override
- public void deallocateFrames(int bytes) {
- // TODO Auto-generated method stub
-
- }
+ return new OnDiskInvertedIndexAccessor(this,
+ (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
@Override
@@ -576,19 +548,21 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
fieldPermutation[i] = i;
}
PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
+ IIndexOperationContext opCtx = new OnDiskInvertedIndexOpContext(btree);
// Search key for finding an inverted-list in the actual index.
ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
ArrayTupleReference prevTuple = new ArrayTupleReference();
IInvertedIndexAccessor invIndexAccessor = createAccessor(NoOpIndexAccessParameters.INSTANCE);
try {
- IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
+ InvertedListCursor invListCursor = createInvertedListRangeSearchCursor();
MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
while (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
// Validate inverted list by checking that the elements are totally ordered.
- invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
- invListCursor.pinPages();
+ openInvertedListCursor(invListCursor, tokenTuple, opCtx);
+ invListCursor.prepareLoadPages();
+ invListCursor.loadPages();
try {
if (invListCursor.hasNext()) {
invListCursor.next();
@@ -607,7 +581,8 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
}
} finally {
- invListCursor.unpinPages();
+ invListCursor.unloadPages();
+ invListCursor.close();
}
}
} finally {
@@ -664,4 +639,5 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
bufferCache.purgeHandle(fileId);
fileId = -1;
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
index 89d4e9a..267cc79 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
@@ -39,7 +39,7 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
private MultiComparator prefixSearchCmp;
private boolean destroyed = false;
- public OnDiskInvertedIndexOpContext(BTree btree) {
+ public OnDiskInvertedIndexOpContext(BTree btree) throws HyracksDataException {
// TODO: Ignore opcallbacks for now.
btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
btreeCursor = btreeAccessor.createSearchCursor(false);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index 7af35ff..a33b045 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -27,7 +27,7 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessor;
@@ -43,8 +43,8 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
private final IIndexAccessor btreeAccessor;
private final IInPlaceInvertedIndex invIndex;
private final IIndexOperationContext opCtx;
- private final IInvertedListCursor invListCursor;
- private boolean unpinNeeded;
+ private final InvertedListCursor invListRangeSearchCursor;
+ private boolean isInvListCursorOpen;
private final IIndexCursor btreeCursor;
private RangePredicate btreePred;
@@ -52,7 +52,8 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
private final PermutingTupleReference tokenTuple;
private ConcatenatingTupleReference concatTuple;
- public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex invIndex, IIndexOperationContext opCtx) {
+ public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex invIndex, IIndexOperationContext opCtx)
+ throws HyracksDataException {
this.btree = ((OnDiskInvertedIndex) invIndex).getBTree();
this.btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
this.invIndex = invIndex;
@@ -65,64 +66,59 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
tokenTuple = new PermutingTupleReference(fieldPermutation);
btreeCursor = btreeAccessor.createSearchCursor(false);
concatTuple = new ConcatenatingTupleReference(2);
- invListCursor = invIndex.createInvertedListCursor();
- unpinNeeded = false;
+ invListRangeSearchCursor = invIndex.createInvertedListRangeSearchCursor();
+ isInvListCursorOpen = false;
}
@Override
public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
this.btreePred = (RangePredicate) searchPred;
btreeAccessor.search(btreeCursor, btreePred);
- invListCursor.pinPages();
- unpinNeeded = true;
+ openInvListRangeSearchCursor();
}
@Override
public boolean doHasNext() throws HyracksDataException {
- if (invListCursor.hasNext()) {
- return true;
- }
- if (unpinNeeded) {
- invListCursor.unpinPages();
- unpinNeeded = false;
- }
- if (!btreeCursor.hasNext()) {
+ // No more results possible
+ if (!isInvListCursorOpen) {
return false;
}
- btreeCursor.next();
- tokenTuple.reset(btreeCursor.getTuple());
- invIndex.openInvertedListCursor(invListCursor, tokenTuple, opCtx);
- invListCursor.pinPages();
- invListCursor.hasNext();
- unpinNeeded = true;
- concatTuple.reset();
- concatTuple.addTuple(tokenTuple);
- return true;
+ if (invListRangeSearchCursor.hasNext()) {
+ return true;
+ }
+ // The current inverted-list-range-search cursor is exhausted.
+ invListRangeSearchCursor.unloadPages();
+ invListRangeSearchCursor.close();
+ isInvListCursorOpen = false;
+ openInvListRangeSearchCursor();
+ return isInvListCursorOpen;
}
@Override
public void doNext() throws HyracksDataException {
- invListCursor.next();
+ invListRangeSearchCursor.next();
if (concatTuple.hasMaxTuples()) {
concatTuple.removeLastTuple();
}
- concatTuple.addTuple(invListCursor.getTuple());
+ concatTuple.addTuple(invListRangeSearchCursor.getTuple());
}
@Override
public void doDestroy() throws HyracksDataException {
- if (unpinNeeded) {
- invListCursor.unpinPages();
- unpinNeeded = false;
+ if (isInvListCursorOpen) {
+ invListRangeSearchCursor.unloadPages();
+ invListRangeSearchCursor.destroy();
+ isInvListCursorOpen = false;
}
btreeCursor.destroy();
}
@Override
public void doClose() throws HyracksDataException {
- if (unpinNeeded) {
- invListCursor.unpinPages();
- unpinNeeded = false;
+ if (isInvListCursorOpen) {
+ invListRangeSearchCursor.unloadPages();
+ invListRangeSearchCursor.close();
+ isInvListCursorOpen = false;
}
btreeCursor.close();
}
@@ -131,4 +127,20 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
public ITupleReference doGetTuple() {
return concatTuple;
}
+
+ // Opens an inverted-list-scan cursor for the given tuple.
+ private void openInvListRangeSearchCursor() throws HyracksDataException {
+ if (btreeCursor.hasNext()) {
+ btreeCursor.next();
+ tokenTuple.reset(btreeCursor.getTuple());
+ invIndex.openInvertedListCursor(invListRangeSearchCursor, tokenTuple, opCtx);
+ invListRangeSearchCursor.prepareLoadPages();
+ invListRangeSearchCursor.loadPages();
+ concatTuple.reset();
+ concatTuple.addTuple(tokenTuple);
+ isInvListCursorOpen = true;
+ } else {
+ isInvListCursorOpen = false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
index 0563ec9..4c521fd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
@@ -19,94 +19,55 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
+/**
+ * A search cursor that fetches the result from an IInvertedIndexSearcher instance.
+ */
public class OnDiskInvertedIndexSearchCursor extends EnforcedIndexCursor {
- private List<ByteBuffer> resultBuffers;
- private int numResultBuffers;
- private int currentBufferIndex = 0;
- private int tupleIndex = 0;
private final IInvertedIndexSearcher invIndexSearcher;
- private final IFrameTupleAccessor fta;
- private final FixedSizeTupleReference frameTuple;
- private final PermutingTupleReference resultTuple;
- public OnDiskInvertedIndexSearchCursor(IInvertedIndexSearcher invIndexSearcher, int numInvListFields) {
+ public OnDiskInvertedIndexSearchCursor(IInvertedIndexSearcher invIndexSearcher) {
this.invIndexSearcher = invIndexSearcher;
- this.fta = invIndexSearcher.createResultFrameTupleAccessor();
- this.frameTuple = (FixedSizeTupleReference) invIndexSearcher.createResultFrameTupleReference();
- // Project away the occurrence count from the result tuples.
- int[] fieldPermutation = new int[numInvListFields];
- for (int i = 0; i < numInvListFields; i++) {
- fieldPermutation[i] = i;
- }
- resultTuple = new PermutingTupleReference(fieldPermutation);
}
@Override
public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
- currentBufferIndex = 0;
- tupleIndex = 0;
- resultBuffers = invIndexSearcher.getResultBuffers();
- numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
- if (numResultBuffers > 0) {
- fta.reset(resultBuffers.get(0));
- }
+ // No-op for this cursor since all necessary information is already set in the given searcher.
+ // This class is just a wrapper.
}
@Override
- public boolean doHasNext() {
- if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount()) {
- return true;
- } else {
- return false;
- }
+ public boolean doHasNext() throws HyracksDataException {
+ return invIndexSearcher.hasNext();
}
@Override
- public void doNext() {
- frameTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
- resultTuple.reset(frameTuple);
- tupleIndex++;
- if (tupleIndex >= fta.getTupleCount()) {
- if (currentBufferIndex + 1 < numResultBuffers) {
- currentBufferIndex++;
- fta.reset(resultBuffers.get(currentBufferIndex));
- tupleIndex = 0;
- }
- }
+ public void doNext() throws HyracksDataException {
+ invIndexSearcher.next();
}
@Override
public ITupleReference doGetTuple() {
- return resultTuple;
+ return invIndexSearcher.getTuple();
}
@Override
- public void doClose() {
- currentBufferIndex = 0;
- tupleIndex = 0;
- invIndexSearcher.reset();
- resultBuffers = invIndexSearcher.getResultBuffers();
- numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
+ public void doClose() throws HyracksDataException {
+ doDestroy();
}
@Override
public void doDestroy() throws HyracksDataException {
- currentBufferIndex = 0;
- tupleIndex = 0;
- resultBuffers = null;
- numResultBuffers = 0;
+ if (invIndexSearcher != null) {
+ invIndexSearcher.destroy();
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index 064a26d..eff4f5a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -19,23 +19,26 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
-import java.util.List;
-
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.data.std.primitive.ShortPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IPartitionedInvertedIndex;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartitions;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implements IPartitionedInvertedIndex {
@@ -51,21 +54,39 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen
}
public class PartitionedOnDiskInvertedIndexAccessor extends OnDiskInvertedIndexAccessor {
- public PartitionedOnDiskInvertedIndexAccessor(OnDiskInvertedIndex index) throws HyracksDataException {
- super(index, new PartitionedTOccurrenceSearcher(ctx, index));
+ public PartitionedOnDiskInvertedIndexAccessor(OnDiskInvertedIndex index, IHyracksTaskContext ctx)
+ throws HyracksDataException {
+ super(index, ctx);
+ }
+
+ @Override
+ public IIndexCursor createSearchCursor(boolean exclusive) throws HyracksDataException {
+ if (searcher == null) {
+ searcher = new PartitionedTOccurrenceSearcher(index, ctx);
+ }
+ return new OnDiskInvertedIndexSearchCursor(searcher);
+ }
+
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException {
+ if (searcher == null) {
+ searcher = new PartitionedTOccurrenceSearcher(index, ctx);
+ }
+ searcher.search(cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
}
}
@Override
public PartitionedOnDiskInvertedIndexAccessor createAccessor(IIndexAccessParameters iap)
throws HyracksDataException {
- return new PartitionedOnDiskInvertedIndexAccessor(this);
+ return new PartitionedOnDiskInvertedIndexAccessor(this,
+ (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
@Override
public boolean openInvertedListPartitionCursors(IInvertedIndexSearcher searcher, IIndexOperationContext ictx,
- short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions,
- List<IInvertedListCursor> cursorsOrderedByTokens) throws HyracksDataException {
+ short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions)
+ throws HyracksDataException {
PartitionedTOccurrenceSearcher partSearcher = (PartitionedTOccurrenceSearcher) searcher;
OnDiskInvertedIndexOpContext ctx = (OnDiskInvertedIndexOpContext) ictx;
ITupleReference lowSearchKey = null;
@@ -95,9 +116,8 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen
ITupleReference btreeTuple = ctx.getBtreeCursor().getTuple();
short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD),
btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
- IInvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor();
- resetInvertedListCursor(btreeTuple, invListCursor);
- cursorsOrderedByTokens.add(invListCursor);
+ InvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor();
+ openInvertedListCursor(btreeTuple, invListCursor);
invListPartitions.addInvertedListCursor(invListCursor, numTokens);
tokenExists = true;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java
index 294eb04..ff9a2f1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java
@@ -25,22 +25,25 @@ import java.util.List;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppenderAccessor;
import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
+import org.apache.hyracks.dataflow.std.buffermanager.BufferManagerBackedVSizeFrame;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IObjectFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
@@ -57,10 +60,12 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc
protected final int OBJECT_CACHE_INIT_SIZE = 10;
protected final int OBJECT_CACHE_EXPAND_SIZE = 10;
- protected final IHyracksCommonContext ctx;
+ protected final IHyracksTaskContext ctx;
protected final InvertedListMerger invListMerger;
- protected final SearchResult searchResult;
+ // Final search result is needed because multiple merge() calls can happen.
+ // We can't just use one of intermediate results as the final search result.
+ protected final InvertedIndexFinalSearchResult finalSearchResult;
protected final IInPlaceInvertedIndex invIndex;
protected final MultiComparator invListCmp;
@@ -71,28 +76,51 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc
protected int occurrenceThreshold;
- protected final IObjectFactory<IInvertedListCursor> invListCursorFactory;
- protected final ObjectCache<IInvertedListCursor> invListCursorCache;
+ protected final IObjectFactory<InvertedListCursor> invListCursorFactory;
+ protected final ObjectCache<InvertedListCursor> invListCursorCache;
- public AbstractTOccurrenceSearcher(IHyracksCommonContext ctx, IInPlaceInvertedIndex invIndex)
+ protected final ISimpleFrameBufferManager bufferManager;
+ protected boolean isFinishedSearch;
+
+ // For a single inverted list case
+ protected InvertedListCursor singleInvListCursor;
+ protected boolean isSingleInvertedList;
+
+ // To read the final search result
+ protected ByteBuffer searchResultBuffer;
+ protected int searchResultTupleIndex = 0;
+ protected final IFrameTupleAccessor searchResultFta;
+ protected FixedSizeTupleReference searchResultTuple;
+
+ public AbstractTOccurrenceSearcher(IInPlaceInvertedIndex invIndex, IHyracksTaskContext ctx)
throws HyracksDataException {
- this.ctx = ctx;
- this.invListMerger = new InvertedListMerger(ctx, invIndex);
- this.searchResult = new SearchResult(invIndex.getInvListTypeTraits(), ctx);
this.invIndex = invIndex;
+ this.ctx = ctx;
+ if (ctx == null) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_HYRACKS_TASK_IS_NULL);
+ }
+ this.bufferManager = TaskUtil.get(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, ctx);
+ if (bufferManager == null) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL);
+ }
+ this.finalSearchResult =
+ new InvertedIndexFinalSearchResult(invIndex.getInvListTypeTraits(), ctx, bufferManager);
+ this.invListMerger = new InvertedListMerger(ctx, invIndex, bufferManager);
this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
- this.invListCursorFactory = new InvertedListCursorFactory(invIndex);
+ this.invListCursorFactory = new InvertedListCursorFactory(invIndex, ctx);
this.invListCursorCache =
new ObjectCache<>(invListCursorFactory, OBJECT_CACHE_INIT_SIZE, OBJECT_CACHE_EXPAND_SIZE);
- this.queryTokenFrame = new VSizeFrame(ctx);
+ this.queryTokenFrame = new BufferManagerBackedVSizeFrame(ctx, bufferManager);
+ if (queryTokenFrame.getBuffer() == null) {
+ throw HyracksDataException.create(ErrorCode.NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH,
+ this.getClass().getSimpleName());
+ }
this.queryTokenAppender = new FrameTupleAppenderAccessor(QUERY_TOKEN_REC_DESC);
this.queryTokenAppender.reset(queryTokenFrame, true);
- }
-
- @Override
- public void reset() {
- searchResult.clear();
- invListMerger.reset();
+ this.isSingleInvertedList = false;
+ this.searchResultTuple = new FixedSizeTupleReference(invIndex.getInvListTypeTraits());
+ this.searchResultFta =
+ new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), invIndex.getInvListTypeTraits());
}
protected void tokenizeQuery(InvertedIndexSearchPredicate searchPred) throws HyracksDataException {
@@ -100,7 +128,7 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc
int queryFieldIndex = searchPred.getQueryFieldIndex();
IBinaryTokenizer queryTokenizer = searchPred.getQueryTokenizer();
// Is this a full-text query?
- // Then, the last argument is conjuctive or disjunctive search option, not a query text.
+ // Then, the last argument is conjunctive or disjunctive search option, not a query text.
// Thus, we need to remove the last argument.
boolean isFullTextSearchQuery = searchPred.getIsFullTextSearchQuery();
// Get the type of query tokenizer.
@@ -144,33 +172,13 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc
}
}
- @Override
- public IFrameTupleAccessor createResultFrameTupleAccessor() {
- return new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), searchResult.getTypeTraits());
- }
-
- @Override
- public ITupleReference createResultFrameTupleReference() {
- return new FixedSizeTupleReference(searchResult.getTypeTraits());
- }
-
- @Override
- public List<ByteBuffer> getResultBuffers() {
- return searchResult.getBuffers();
- }
-
- @Override
- public int getNumValidResultBuffers() {
- return searchResult.getCurrentBufferIndex() + 1;
- }
-
public int getOccurrenceThreshold() {
return occurrenceThreshold;
}
public void printNewResults(int maxResultBufIdx, List<ByteBuffer> buffer) {
StringBuffer strBuffer = new StringBuffer();
- FixedSizeFrameTupleAccessor resultFrameTupleAcc = searchResult.getAccessor();
+ FixedSizeFrameTupleAccessor resultFrameTupleAcc = finalSearchResult.getAccessor();
for (int i = 0; i <= maxResultBufIdx; i++) {
ByteBuffer testBuf = buffer.get(i);
resultFrameTupleAcc.reset(testBuf);
@@ -183,4 +191,99 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc
}
System.out.println(strBuffer.toString());
}
+
+ /**
+ * Checks whether underlying the inverted list cursor or final search result has a tuple to return.
+ */
+ @Override
+ public boolean hasNext() throws HyracksDataException {
+ do {
+ boolean moreToRead = hasMoreElement();
+ if (moreToRead) {
+ return true;
+ }
+ // Current cursor or buffer is exhausted. Unbinds the inverted list cursor or
+ // cleans the output buffer of the final search result.
+ resetResultSource();
+ // Search is done? Then, there's nothing left.
+ if (isFinishedSearch) {
+ return false;
+ }
+ // Otherwise, resume the search process.
+ continueSearch();
+ } while (true);
+ }
+
+ @Override
+ public void next() throws HyracksDataException {
+ // Case 1: fetching a tuple from an inverted list cursor
+ if (isSingleInvertedList) {
+ singleInvListCursor.next();
+ } else {
+ // Case 2: fetching a tuple from the output frame of a final search result
+ searchResultTuple.reset(searchResultFta.getBuffer().array(),
+ searchResultFta.getTupleStartOffset(searchResultTupleIndex));
+ searchResultTupleIndex++;
+ }
+ }
+
+ private boolean hasMoreElement() throws HyracksDataException {
+ // Case #1: single inverted list cursor
+ if (isSingleInvertedList) {
+ return singleInvListCursor.hasNext();
+ }
+ // Case #2: ouput buffer from a final search result
+ return searchResultTupleIndex < searchResultFta.getTupleCount();
+ }
+
+ private void resetResultSource() throws HyracksDataException {
+ if (isSingleInvertedList) {
+ isSingleInvertedList = false;
+ singleInvListCursor.unloadPages();
+ singleInvListCursor.close();
+ singleInvListCursor = null;
+ } else {
+ finalSearchResult.resetBuffer();
+ searchResultTupleIndex = 0;
+ }
+ }
+
+ public void destroy() throws HyracksDataException {
+ // To ensure to release the buffer of the query token frame.
+ ((BufferManagerBackedVSizeFrame) queryTokenFrame).destroy();
+
+ // Releases the frames of the cursor.
+ if (isSingleInvertedList && singleInvListCursor != null) {
+ singleInvListCursor.unloadPages();
+ singleInvListCursor.close();
+ }
+ // Releases the frame of the final search result.
+ finalSearchResult.close();
+
+ // Releases the frames of the two intermediate search result.
+ invListMerger.close();
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ if (isSingleInvertedList) {
+ return singleInvListCursor.getTuple();
+ }
+ return searchResultTuple;
+ }
+
+ /**
+ * Prepares the search process. This mainly allocates/clears the buffer frames of the each component.
+ */
+ protected void prepareSearch() throws HyracksDataException {
+ finalSearchResult.prepareIOBuffer();
+ invListMerger.prepareMerge();
+ ((BufferManagerBackedVSizeFrame) queryTokenFrame).acquireFrame();
+ isFinishedSearch = false;
+ isSingleInvertedList = false;
+ searchResultFta.reset(finalSearchResult.getNextFrame());
+ searchResultTupleIndex = 0;
+ singleInvListCursor = null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexFinalSearchResult.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexFinalSearchResult.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexFinalSearchResult.java
new file mode 100644
index 0000000..55acb33
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexFinalSearchResult.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.search;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
+
+/**
+ * This is an in-memory based storage for final results of inverted-index searches.
+ * Only one frame is used at a time. The same frame will be used multiple times.
+ */
+public class InvertedIndexFinalSearchResult extends InvertedIndexSearchResult {
+
+ public InvertedIndexFinalSearchResult(ITypeTraits[] invListFields, IHyracksTaskContext ctx,
+ ISimpleFrameBufferManager bufferManager) throws HyracksDataException {
+ super(invListFields, ctx, bufferManager);
+ }
+
+ /**
+ * The final search result only needs to keep the inverted list fields, not its count.
+ */
+ @Override
+ protected void initTypeTraits(ITypeTraits[] invListFields) {
+ typeTraits = new ITypeTraits[invListFields.length];
+ int tmp = 0;
+ for (int i = 0; i < invListFields.length; i++) {
+ typeTraits[i] = invListFields[i];
+ tmp += invListFields[i].getFixedLength();
+ }
+ invListElementSize = tmp;
+ }
+
+ /**
+ * Prepares the write operation. A result of the final search result will be always in memory.
+ */
+ @Override
+ public void prepareWrite(int numExpectedPages) throws HyracksDataException {
+ // Final search result: we will use the ioBuffer and we will not create any file.
+ // This method can be called multiple times in case of the partitioned T-Occurrence search.
+ // For those cases, if the write process has already begun, we should not clear the buffer.
+ isInMemoryOpMode = true;
+ isFileOpened = false;
+ resetAppenderLocation(IO_BUFFER_IDX);
+ isWriteFinished = false;
+ }
+
+ /**
+ * Appends an element to the frame of this result. When processing the final list,
+ * it does not create an additional frame when a frame becomes full to let the caller consume the frame.
+ *
+ * @return false if the current frame for the final result is full.
+ * true otherwise.
+ */
+ @Override
+ public boolean append(ITupleReference invListElement, int count) throws HyracksDataException {
+ // Pauses the addition of this tuple if the current page is full.
+ if (!appender.hasSpace()) {
+ return false;
+ }
+ // Appends the given inverted-list element.
+ if (!appender.append(invListElement.getFieldData(0), invListElement.getFieldStart(0), invListElementSize)) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT);
+ }
+ appender.incrementTupleCount(1);
+ numResults++;
+
+ return true;
+ }
+
+ /**
+ * Finalizes the write operation.
+ */
+ @Override
+ public void finalizeWrite() throws HyracksDataException {
+ if (isWriteFinished) {
+ return;
+ }
+ isWriteFinished = true;
+ }
+
+ /**
+ * Prepares a read operation.
+ */
+ @Override
+ public void prepareResultRead() throws HyracksDataException {
+ if (isInReadMode) {
+ return;
+ }
+ currentReaderBufIdx = 0;
+ isInReadMode = true;
+ }
+
+ /**
+ * Gets the next frame of the current result file.
+ */
+ @Override
+ public ByteBuffer getNextFrame() throws HyracksDataException {
+ return buffers.get(IO_BUFFER_IDX);
+ }
+
+ /**
+ * Finishes reading the result and frees the buffer.
+ */
+ @Override
+ public void closeResultRead(boolean deallocateIOBufferNeeded) throws HyracksDataException {
+ // Deallocates I/O buffer if requested.
+ if (deallocateIOBufferNeeded) {
+ deallocateIOBuffer();
+ }
+ }
+
+ /**
+ * Deallocates the buffer.
+ */
+ @Override
+ public void close() throws HyracksDataException {
+ deallocateIOBuffer();
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ // Resets the I/O buffer.
+ clearBuffer(ioBuffer);
+
+ searchResultWriter = null;
+ searchResultReader = null;
+ isInReadMode = false;
+ isWriteFinished = false;
+ isInMemoryOpMode = false;
+ isFileOpened = false;
+ currentWriterBufIdx = 0;
+ currentReaderBufIdx = 0;
+ numResults = 0;
+ }
+
+ /**
+ * Deallocates the I/O buffer (one frame). This should be the last operation.
+ */
+ @Override
+ protected void deallocateIOBuffer() throws HyracksDataException {
+ if (ioBufferFrame != null) {
+ bufferManager.releaseFrame(ioBuffer);
+ buffers.clear();
+ ioBufferFrame = null;
+ ioBuffer = null;
+ }
+ }
+
+ /**
+ * Resets the buffer.
+ */
+ public void resetBuffer() {
+ appender.reset(buffers.get(IO_BUFFER_IDX));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchResult.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchResult.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchResult.java
new file mode 100644
index 0000000..527d624
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchResult.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.search;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ListIterator;
+
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.io.RunFileReader;
+import org.apache.hyracks.dataflow.common.io.RunFileWriter;
+import org.apache.hyracks.dataflow.std.buffermanager.BufferManagerBackedVSizeFrame;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
+import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAccessor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAppender;
+import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeTupleReference;
+
+/**
+ * Disk-based or in-memory based storage for intermediate and final results of inverted-index
+ * searches. One frame is dedicated to I/O operation for disk operation mode.
+ */
+public class InvertedIndexSearchResult {
+ // The size of count field for each element. Currently, we use an integer value.
+ protected static final int ELEMENT_COUNT_SIZE = 4;
+ // I/O buffer's index in the buffers
+ protected static final int IO_BUFFER_IDX = 0;
+ protected static final String FILE_PREFIX = "InvertedIndexSearchResult";
+ protected final IHyracksTaskContext ctx;
+ protected final FixedSizeFrameTupleAppender appender;
+ protected final FixedSizeFrameTupleAccessor accessor;
+ protected final FixedSizeTupleReference tuple;
+ protected final ISimpleFrameBufferManager bufferManager;
+ protected ITypeTraits[] typeTraits;
+ protected int invListElementSize;
+
+ protected int currentWriterBufIdx;
+ protected int currentReaderBufIdx;
+ protected int numResults;
+ protected int numPossibleElementPerPage;
+ // Read and Write I/O buffer
+ protected IFrame ioBufferFrame = null;
+ protected ByteBuffer ioBuffer = null;
+ // Buffers for in-memory operation mode. The first buffer is the ioBuffer.
+ // In case of the final search result, we will use only use the first buffer. No file will be created.
+ protected ArrayList<ByteBuffer> buffers;
+
+ protected RunFileWriter searchResultWriter;
+ protected RunFileReader searchResultReader;
+ protected boolean isInMemoryOpMode;
+ protected boolean isInReadMode;
+ protected boolean isWriteFinished;
+ protected boolean isFileOpened;
+
+ public InvertedIndexSearchResult(ITypeTraits[] invListFields, IHyracksTaskContext ctx,
+ ISimpleFrameBufferManager bufferManager) throws HyracksDataException {
+ initTypeTraits(invListFields);
+ this.ctx = ctx;
+ appender = new FixedSizeFrameTupleAppender(ctx.getInitialFrameSize(), typeTraits);
+ accessor = new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), typeTraits);
+ tuple = new FixedSizeTupleReference(typeTraits);
+ this.bufferManager = bufferManager;
+ this.isInReadMode = false;
+ this.isWriteFinished = false;
+ this.isInMemoryOpMode = false;
+ this.isFileOpened = false;
+ this.ioBufferFrame = null;
+ this.ioBuffer = null;
+ this.buffers = null;
+ this.currentWriterBufIdx = 0;
+ this.currentReaderBufIdx = 0;
+ this.numResults = 0;
+ calculateNumElementPerPage();
+ // Allocates one frame for read/write operation.
+ prepareIOBuffer();
+ }
+
+ /**
+ * Initializes the element type in the search result. In addition to the element, we will keep one more integer
+ * per element to keep its occurrence count.
+ */
+ protected void initTypeTraits(ITypeTraits[] invListFields) {
+ typeTraits = new ITypeTraits[invListFields.length + 1];
+ int tmp = 0;
+ for (int i = 0; i < invListFields.length; i++) {
+ typeTraits[i] = invListFields[i];
+ tmp += invListFields[i].getFixedLength();
+ }
+ invListElementSize = tmp;
+ // Integer for counting occurrences.
+ typeTraits[invListFields.length] = IntegerPointable.TYPE_TRAITS;
+ }
+
+ /**
+ * Prepares the write operation. Tries to allocate buffers for the expected number of pages.
+ * If that is possible, all operations will be executed in memory.
+ * If not, all operations will use a file on disk except for the final search result.
+ * A result of the final search result will be always in memory.
+ */
+ public void prepareWrite(int numExpectedPages) throws HyracksDataException {
+ if (isInReadMode || isWriteFinished || searchResultWriter != null) {
+ return;
+ }
+ // Intermediate results? disk or in-memory based
+ // Allocates more buffers.
+ isInMemoryOpMode = tryAllocateBuffers(numExpectedPages);
+ if (!isInMemoryOpMode) {
+ // Not enough number of buffers. Switch to the file I/O mode.
+ createAndOpenFile();
+ }
+ appender.reset(ioBuffer);
+ isWriteFinished = false;
+ }
+
+ /**
+ * Appends an element and its count to the current frame of this result. The boolean value is necessary for
+ * the final search result case since the append() of that class is overriding this method.
+ */
+ public boolean append(ITupleReference invListElement, int count) throws HyracksDataException {
+ ByteBuffer currentBuffer;
+ // Moves to the next page if the current page is full.
+ if (!appender.hasSpace()) {
+ currentWriterBufIdx++;
+ if (isInMemoryOpMode) {
+ currentBuffer = buffers.get(currentWriterBufIdx);
+ } else {
+ searchResultWriter.nextFrame(ioBuffer);
+ currentBuffer = ioBuffer;
+ }
+ appender.reset(currentBuffer);
+ }
+ // Appends inverted-list element.
+ if (!appender.append(invListElement.getFieldData(0), invListElement.getFieldStart(0), invListElementSize)) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT);
+ }
+ // Appends count.
+ if (!appender.append(count)) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT);
+ }
+ appender.incrementTupleCount(1);
+ numResults++;
+
+ // Always true for the intermediate result. An append should not fail.
+ return true;
+ }
+
+ /**
+ * Finalizes the write operation. After this, no more write operation can be conducted.
+ */
+ public void finalizeWrite() throws HyracksDataException {
+ if (isWriteFinished) {
+ return;
+ }
+ // For in-memory operation (including the final result), no specific operations are required.
+ // For disk-based operation, needs to close the writer.
+ if (!isInMemoryOpMode && searchResultWriter != null) {
+ searchResultWriter.nextFrame(ioBuffer);
+ searchResultWriter.close();
+ }
+ isWriteFinished = true;
+ }
+
+ /**
+ * Prepares a read operation.
+ */
+ public void prepareResultRead() throws HyracksDataException {
+ if (isInReadMode) {
+ return;
+ }
+ // No specific operation is required for in-memory mode (including the final result).
+ if (!isInMemoryOpMode && searchResultWriter != null) {
+ if (!isWriteFinished) {
+ finalizeWrite();
+ }
+ searchResultReader = searchResultWriter.createDeleteOnCloseReader();
+ searchResultReader.open();
+ searchResultReader.setDeleteAfterClose(true);
+ }
+ currentReaderBufIdx = 0;
+ isInReadMode = true;
+ }
+
+ /**
+ * Gets the next frame of the current result file. A caller should make sure that initResultRead() is called first.
+ */
+ public ByteBuffer getNextFrame() throws HyracksDataException {
+ ByteBuffer returnedBuffer = null;
+ if (isInMemoryOpMode) {
+ // In-memory mode for an intermediate search result
+ returnedBuffer = buffers.get(currentReaderBufIdx);
+ currentReaderBufIdx++;
+ } else if (searchResultReader != null && searchResultReader.nextFrame(ioBufferFrame)) {
+ // Disk-based mode for an intermediate search result
+ returnedBuffer = ioBufferFrame.getBuffer();
+ }
+ return returnedBuffer;
+ }
+
+ /**
+ * Finishes reading the result and frees the buffer.
+ */
+ public void closeResultRead(boolean deallocateIOBufferNeeded) throws HyracksDataException {
+ if (isInMemoryOpMode) {
+ // In-memory mode? Releases all buffers for an intermediate search result.
+ deallocateBuffers();
+ } else if (searchResultReader != null) {
+ // Disk mode? Closes the file handle (this should delete the file also.)
+ searchResultReader.close();
+ }
+
+ // Deallocates I/O buffer if requested.
+ if (deallocateIOBufferNeeded) {
+ deallocateIOBuffer();
+ }
+ }
+
+ public int getCurrentBufferIndex() {
+ return currentWriterBufIdx;
+ }
+
+ public ITypeTraits[] getTypeTraits() {
+ return typeTraits;
+ }
+
+ public int getNumResults() {
+ return numResults;
+ }
+
+ /**
+ * Deletes any associated file and deallocates all buffers.
+ */
+ public void close() throws HyracksDataException {
+ if (isInMemoryOpMode) {
+ deallocateBuffers();
+ } else {
+ if (searchResultReader != null) {
+ searchResultReader.close();
+ } else if (searchResultWriter != null) {
+ searchResultWriter.erase();
+ }
+ }
+ deallocateIOBuffer();
+ }
+
+ public void reset() throws HyracksDataException {
+ // Removes the file if it was in the disk op mode.
+ if (searchResultReader != null) {
+ searchResultReader.close();
+ } else if (searchResultWriter != null) {
+ searchResultWriter.erase();
+ } else if (buffers.size() > 1) {
+ // In-memory mode? Deallocates all buffers.
+ deallocateBuffers();
+ }
+
+ // Resets the I/O buffer.
+ clearBuffer(ioBuffer);
+
+ searchResultWriter = null;
+ searchResultReader = null;
+ isInReadMode = false;
+ isWriteFinished = false;
+ isInMemoryOpMode = false;
+ isFileOpened = false;
+ currentWriterBufIdx = 0;
+ currentReaderBufIdx = 0;
+ numResults = 0;
+ }
+
+ /**
+ * Gets the expected number of pages if all elements are created as a result.
+ * An assumption is that there are no common elements between the previous result and the cursor.
+ */
+ public int getExpectedNumPages(int numExpectedElements) {
+ return (int) Math.ceil((double) numExpectedElements / numPossibleElementPerPage);
+ }
+
+ // Gets the number of possible elements per page based on the inverted list element size.
+ protected void calculateNumElementPerPage() {
+ int frameSize = ctx.getInitialFrameSize();
+ // The count of Minframe, and the count of tuples in a frame should be deducted.
+ frameSize = frameSize - FixedSizeFrameTupleAppender.MINFRAME_COUNT_SIZE
+ - FixedSizeFrameTupleAppender.TUPLE_COUNT_SIZE;
+ numPossibleElementPerPage = (int) Math.floor((double) frameSize / (invListElementSize + ELEMENT_COUNT_SIZE));
+ }
+
+ /**
+ * Allocates the buffer for read/write operation and initializes the buffers array that will be used keep a result.
+ */
+ protected void prepareIOBuffer() throws HyracksDataException {
+ if (ioBufferFrame != null) {
+ clearBuffer(ioBuffer);
+ } else {
+ ioBufferFrame = new BufferManagerBackedVSizeFrame(ctx, bufferManager);
+ ioBuffer = ioBufferFrame.getBuffer();
+ if (ioBuffer == null) {
+ // One frame should be allocated for conducting read/write
+ // operation. Otherwise, can't store the result.
+ throw HyracksDataException.create(ErrorCode.NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH,
+ this.getClass().getSimpleName());
+ }
+ clearBuffer(ioBuffer);
+ // For keeping the results in memory if possible.
+ buffers = new ArrayList<ByteBuffer>();
+ buffers.add(ioBuffer);
+ }
+ }
+
+ /**
+ * Tries to allocate buffers to accommodate the results in memory.
+ */
+ protected boolean tryAllocateBuffers(int numExpectedPages) throws HyracksDataException {
+ boolean allBufferAllocated = true;
+ while (buffers.size() < numExpectedPages) {
+ ByteBuffer tmpBuffer = bufferManager.acquireFrame(ctx.getInitialFrameSize());
+ if (tmpBuffer == null) {
+ // Budget exhausted
+ allBufferAllocated = false;
+ break;
+ } else {
+ clearBuffer(tmpBuffer);
+ }
+ buffers.add(tmpBuffer);
+ }
+ return allBufferAllocated;
+ }
+
+ // Creates a file for the writer.
+ protected void createAndOpenFile() throws HyracksDataException {
+ if (isInMemoryOpMode) {
+ // In-memory mode should not generate a file.
+ return;
+ }
+ if (searchResultWriter == null) {
+ FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(FILE_PREFIX);
+ searchResultWriter = new RunFileWriter(file, ctx.getIoManager());
+ searchResultWriter.open();
+ isFileOpened = true;
+ }
+ }
+
+ // Deallocates the I/O buffer (one frame). This should be the last oepration.
+ protected void deallocateIOBuffer() throws HyracksDataException {
+ if (ioBufferFrame != null) {
+ bufferManager.releaseFrame(ioBuffer);
+ buffers.clear();
+ ioBufferFrame = null;
+ ioBuffer = null;
+ }
+ }
+
+ /**
+ * Deallocates the buffers. We do not remove the first buffer since it can be used as an I/O buffer.
+ */
+ protected void deallocateBuffers() throws HyracksDataException {
+ int toDeleteCount = buffers.size() - 1;
+ int deletedCount = 0;
+ for (ListIterator<ByteBuffer> iter = buffers.listIterator(buffers.size()); iter.hasPrevious();) {
+ if (deletedCount >= toDeleteCount) {
+ break;
+ }
+ ByteBuffer next = iter.previous();
+ bufferManager.releaseFrame(next);
+ iter.remove();
+ deletedCount++;
+ }
+ }
+
+ public FixedSizeFrameTupleAccessor getAccessor() {
+ return accessor;
+ }
+
+ public FixedSizeFrameTupleAppender getAppender() {
+ return appender;
+ }
+
+ public FixedSizeTupleReference getTuple() {
+ return tuple;
+ }
+
+ protected void clearBuffer(ByteBuffer bufferToClear) {
+ Arrays.fill(bufferToClear.array(), (byte) 0);
+ bufferToClear.clear();
+ }
+
+ protected void resetAppenderLocation(int bufferIdx) {
+ accessor.reset(buffers.get(bufferIdx));
+ appender.reset(buffers.get(bufferIdx), false, accessor.getTupleCount(),
+ accessor.getTupleEndOffset(accessor.getTupleCount() - 1));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListCursorFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListCursorFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListCursorFactory.java
index 8ed80f6..d0337cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListCursorFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListCursorFactory.java
@@ -19,20 +19,24 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.search;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IObjectFactory;
-public class InvertedListCursorFactory implements IObjectFactory<IInvertedListCursor> {
+public class InvertedListCursorFactory implements IObjectFactory<InvertedListCursor> {
private final IInPlaceInvertedIndex invIndex;
+ private final IHyracksTaskContext ctx;
- public InvertedListCursorFactory(IInPlaceInvertedIndex invIndex) {
+ public InvertedListCursorFactory(IInPlaceInvertedIndex invIndex, IHyracksTaskContext ctx) {
this.invIndex = invIndex;
+ this.ctx = ctx;
}
@Override
- public IInvertedListCursor create() {
- return invIndex.createInvertedListCursor();
+ public InvertedListCursor create() throws HyracksDataException {
+ return invIndex.createInvertedListCursor(ctx);
}
}
[4/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
deleted file mode 100644
index e9b3f21..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedListCursor.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.invertedindex.api;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.common.MultiComparator;
-
-public interface IInvertedListCursor extends Comparable<IInvertedListCursor> {
- public void reset(int startPageId, int endPageId, int startOff, int numElements);
-
- public void pinPages() throws HyracksDataException;
-
- public void unpinPages() throws HyracksDataException;
-
- public boolean hasNext() throws HyracksDataException;
-
- public void next() throws HyracksDataException;
-
- public ITupleReference getTuple();
-
- // getters
- public int size() throws HyracksDataException;
-
- public int getStartPageId();
-
- public int getEndPageId();
-
- public int getStartOff();
-
- public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException;
-
- // for debugging
- @SuppressWarnings("rawtypes")
- public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException;
-
- @SuppressWarnings("rawtypes")
- public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IObjectFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IObjectFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IObjectFactory.java
index 93b182d..fe735bc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IObjectFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IObjectFactory.java
@@ -19,6 +19,8 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public interface IObjectFactory<T> {
- public T create();
+ public T create() throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
index df8e6f0..fd80c00 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IPartitionedInvertedIndex.java
@@ -19,16 +19,14 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
-import java.util.List;
-
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartitions;
public interface IPartitionedInvertedIndex {
public boolean openInvertedListPartitionCursors(IInvertedIndexSearcher searcher, IIndexOperationContext ictx,
- short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions,
- List<IInvertedListCursor> cursorsOrderedByTokens) throws HyracksDataException;
+ short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions)
+ throws HyracksDataException;
public boolean isEmpty();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/InvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/InvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/InvertedListCursor.java
new file mode 100644
index 0000000..9db7dc8
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/InvertedListCursor.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.api;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+/**
+ * A cursor that reads an inverted list.
+ */
+public abstract class InvertedListCursor extends EnforcedIndexCursor implements Comparable<InvertedListCursor> {
+
+ /**
+ * Opens an inverted list cursor.
+ */
+ protected void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ // If the given cursor state has page ids and the number of elements for the given inverted list,
+ // this should be set. Otherwise (for in-memory cursor), doesn't need to do anything.
+ int invListStartPageId = LSMInvertedIndexSearchCursorInitialState.INVALID_VALUE;
+ int invListEndPageId = LSMInvertedIndexSearchCursorInitialState.INVALID_VALUE;
+ int invListStartOffset = LSMInvertedIndexSearchCursorInitialState.INVALID_VALUE;
+ int invListNumElements = LSMInvertedIndexSearchCursorInitialState.INVALID_VALUE;
+ if (initialState != null && initialState instanceof LSMInvertedIndexSearchCursorInitialState) {
+ LSMInvertedIndexSearchCursorInitialState invIndexInitialState =
+ (LSMInvertedIndexSearchCursorInitialState) initialState;
+ invListStartPageId = invIndexInitialState.getInvListStartPageId();
+ invListEndPageId = invIndexInitialState.getInvListEndPageId();
+ invListStartOffset = invIndexInitialState.getInvListStartOffset();
+ invListNumElements = invIndexInitialState.getInvListNumElements();
+ }
+ if (invListNumElements != LSMInvertedIndexSearchCursorInitialState.INVALID_VALUE) {
+ setInvListInfo(invListStartPageId, invListEndPageId, invListStartOffset, invListNumElements);
+ }
+ }
+
+ /**
+ * Sets the disk-based inverted list information such as page ids and the number of elements
+ * for the given inverted list.
+ */
+ protected abstract void setInvListInfo(int startPageId, int endPageId, int startOff, int numElements)
+ throws HyracksDataException;
+
+ /**
+ * Conducts any operation that is required before loading pages.
+ */
+ public abstract void prepareLoadPages() throws HyracksDataException;
+
+ /**
+ * Loads one or more pages to memory.
+ */
+ public abstract void loadPages() throws HyracksDataException;
+
+ /**
+ * Unloads currently loaded pages in the memory.
+ */
+ public abstract void unloadPages() throws HyracksDataException;
+
+ /**
+ * Gets the cardinality of elements in the cursor.
+ */
+ public abstract int size() throws HyracksDataException;
+
+ /**
+ * Checks whether the given tuple is contained in the cursor.
+ */
+ public abstract boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp)
+ throws HyracksDataException;
+
+ /**
+ * Prints all elements in the cursor (debug method).
+ */
+ @SuppressWarnings("rawtypes")
+ public abstract String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException;
+
+ /**
+ * Prints the current element in the cursor (debug method).
+ */
+ @SuppressWarnings("rawtypes")
+ public abstract String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
index 35c0dec..13a649d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
@@ -49,13 +49,16 @@ public class LSMInvertedIndexSearchOperatorDescriptor extends AbstractSingleActi
private final IMissingWriterFactory missingWriterFactory;
private final ISearchOperationCallbackFactory searchCallbackFactory;
private final int numOfFields;
+ // the maximum number of frames that this inverted-index-search can use
+ private final int frameLimit;
public LSMInvertedIndexSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int queryField, IIndexDataflowHelperFactory indexHelperFactory,
IBinaryTokenizerFactory queryTokenizerFactory, IInvertedIndexSearchModifierFactory searchModifierFactory,
boolean retainInput, boolean retainMissing, IMissingWriterFactory missingWriterFactory,
ISearchOperationCallbackFactory searchCallbackFactory, int[] minFilterFieldIndexes,
- int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery, int numOfFields, boolean appendIndexFilter) {
+ int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery, int numOfFields, boolean appendIndexFilter,
+ int frameLimit) {
super(spec, 1, 1);
this.indexHelperFactory = indexHelperFactory;
this.queryTokenizerFactory = queryTokenizerFactory;
@@ -71,6 +74,7 @@ public class LSMInvertedIndexSearchOperatorDescriptor extends AbstractSingleActi
this.appendIndexFilter = appendIndexFilter;
this.numOfFields = numOfFields;
this.outRecDescs[0] = outRecDesc;
+ this.frameLimit = frameLimit;
}
@Override
@@ -81,6 +85,6 @@ public class LSMInvertedIndexSearchOperatorDescriptor extends AbstractSingleActi
recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), partition, minFilterFieldIndexes,
maxFilterFieldIndexes, indexHelperFactory, retainInput, retainMissing, missingWriterFactory,
searchCallbackFactory, searchModifier, queryTokenizerFactory, queryField, isFullTextSearchQuery,
- numOfFields, appendIndexFilter);
+ numOfFields, appendIndexFilter, frameLimit);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
index ed7a4a6..a27dea7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
@@ -23,13 +23,20 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
+import org.apache.hyracks.dataflow.std.buffermanager.DeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.FramePoolBackedFrameBufferManager;
+import org.apache.hyracks.dataflow.std.buffermanager.IDeallocatableFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
public class LSMInvertedIndexSearchOperatorNodePushable extends IndexSearchOperatorNodePushable {
@@ -41,14 +48,17 @@ public class LSMInvertedIndexSearchOperatorNodePushable extends IndexSearchOpera
// Keeps the information whether the given query is a full-text search or not.
// We need to have this information to stop the search process since we don't allow a phrase search yet.
protected final boolean isFullTextSearchQuery;
+ // Budget-constrained buffer manager for conducting the search operation
+ protected final ISimpleFrameBufferManager bufferManagerForSearch;
+ protected final IDeallocatableFramePool framePool;
public LSMInvertedIndexSearchOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc,
int partition, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput, boolean retainMissing,
IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchCallbackFactory,
IInvertedIndexSearchModifier searchModifier, IBinaryTokenizerFactory binaryTokenizerFactory,
- int queryFieldIndex, boolean isFullTextSearchQuery, int numOfFields, boolean appendIndexFilter)
- throws HyracksDataException {
+ int queryFieldIndex, boolean isFullTextSearchQuery, int numOfFields, boolean appendIndexFilter,
+ int frameLimit) throws HyracksDataException {
super(ctx, inputRecDesc, partition, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, appendIndexFilter);
this.searchModifier = searchModifier;
@@ -60,6 +70,11 @@ public class LSMInvertedIndexSearchOperatorNodePushable extends IndexSearchOpera
this.frameTuple = new FrameTupleReference();
}
this.numOfFields = numOfFields;
+ // Intermediate and final search result will use this buffer manager to get frames.
+ this.framePool = new DeallocatableFramePool(ctx, frameLimit * ctx.getInitialFrameSize());
+ this.bufferManagerForSearch = new FramePoolBackedFrameBufferManager(framePool);
+ // Keep the buffer manager in the hyracks context so that the search process can get it via the context.
+ TaskUtil.put(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, bufferManagerForSearch, ctx);
}
@Override
@@ -87,4 +102,9 @@ public class LSMInvertedIndexSearchOperatorNodePushable extends IndexSearchOpera
protected int getFieldCount() {
return numOfFields;
}
+
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ iap.getParameters().put(HyracksConstants.HYRACKS_TASK_CONTEXT, ctx);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 1beff71..a395e67 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -69,8 +69,6 @@ import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -193,6 +191,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
@Override
public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
throws HyracksDataException {
+ LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
int numComponents = operationalComponents.size();
boolean includeMutableComponent = false;
@@ -203,15 +202,13 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
ILSMComponent component = operationalComponents.get(i);
if (component.getType() == LSMComponentType.MEMORY) {
includeMutableComponent = true;
- IIndexAccessor invIndexAccessor =
- component.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ IIndexAccessor invIndexAccessor = component.getIndex().createAccessor(ctx.getIndexAccessParameters());
indexAccessors.add(invIndexAccessor);
IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getBuddyIndex()
.createAccessor(NoOpIndexAccessParameters.INSTANCE);
deletedKeysBTreeAccessors.add(deletedKeysAccessor);
} else {
- IIndexAccessor invIndexAccessor =
- component.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ IIndexAccessor invIndexAccessor = component.getIndex().createAccessor(ctx.getIndexAccessParameters());
indexAccessors.add(invIndexAccessor);
IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexDiskComponent) component).getBuddyIndex()
.createAccessor(NoOpIndexAccessParameters.INSTANCE);
@@ -436,15 +433,13 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
@Override
public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) throws HyracksDataException {
- return new LSMInvertedIndexAccessor(getHarness(),
- createOpContext(iap.getModificationCallback(), iap.getSearchOperationCallback()));
+ return new LSMInvertedIndexAccessor(getHarness(), createOpContext(iap));
}
@Override
- protected LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new LSMInvertedIndexOpContext(this, memoryComponents, modificationCallback, searchCallback,
- invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps, getFilterCmpFactories(), tracer);
+ protected LSMInvertedIndexOpContext createOpContext(IIndexAccessParameters iap) throws HyracksDataException {
+ return new LSMInvertedIndexOpContext(this, memoryComponents, iap, invertedIndexFieldsForNonBulkLoadOps,
+ filterFieldsForNonBulkLoadOps, getFilterCmpFactories(), tracer);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index c33e2ce..d7408ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -35,7 +35,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
@@ -178,12 +178,12 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedInd
}
@Override
- public IInvertedListCursor createInvertedListCursor() {
+ public InvertedListCursor createInvertedListCursor() {
throw new UnsupportedOperationException("Cannot create inverted list cursor on lsm inverted index.");
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException {
throw new UnsupportedOperationException("Cannot open inverted list cursor on lsm inverted index.");
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 1fe4bd2..e7a725e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -30,9 +30,8 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexAccessor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.util.trace.ITracer;
public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext {
@@ -47,21 +46,22 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
private IIndexAccessor[] deletedKeysBTreeAccessors;
private IInvertedIndexAccessor currentMutableInvIndexAccessors;
private IIndexAccessor currentDeletedKeysBTreeAccessors;
+ // To keep the buffer frame manager in case of a search
+ private IIndexAccessParameters iap;
private boolean destroyed = false;
public LSMInvertedIndexOpContext(ILSMIndex index, List<ILSMMemoryComponent> mutableComponents,
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
- int[] invertedIndexFields, int[] filterFields, IBinaryComparatorFactory[] filterComparatorFactories,
- ITracer tracer) throws HyracksDataException {
- super(index, invertedIndexFields, filterFields, filterComparatorFactories, searchCallback, modificationCallback,
- tracer);
+ IIndexAccessParameters iap, int[] invertedIndexFields, int[] filterFields,
+ IBinaryComparatorFactory[] filterComparatorFactories, ITracer tracer) throws HyracksDataException {
+ super(index, invertedIndexFields, filterFields, filterComparatorFactories, iap.getSearchOperationCallback(),
+ iap.getModificationCallback(), tracer);
mutableInvIndexAccessors = new IInvertedIndexAccessor[mutableComponents.size()];
deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()];
for (int i = 0; i < mutableComponents.size(); i++) {
LSMInvertedIndexMemoryComponent mutableComponent =
(LSMInvertedIndexMemoryComponent) mutableComponents.get(i);
if (allFields != null) {
- mutableInvIndexAccessors[i] = mutableComponent.getIndex().createAccessor(allFields);
+ mutableInvIndexAccessors[i] = mutableComponent.getIndex().createAccessor(iap, allFields);
} else {
mutableInvIndexAccessors[i] =
mutableComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
@@ -77,6 +77,7 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
}
keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
+ this.iap = iap;
}
@Override
@@ -97,6 +98,10 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
return currentDeletedKeysBTreeAccessors;
}
+ public IIndexAccessParameters getIndexAccessParameters() {
+ return iap;
+ }
+
@Override
public void destroy() throws HyracksDataException {
if (destroyed) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursorInitialState.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursorInitialState.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursorInitialState.java
index 93ede6d..6e35e07 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursorInitialState.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexSearchCursorInitialState.java
@@ -35,6 +35,8 @@ import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class LSMInvertedIndexSearchCursorInitialState implements ICursorInitialState {
+ public static final int INVALID_VALUE = -1;
+
private final boolean includeMemComponent;
private final ILSMHarness lsmHarness;
private final List<IIndexAccessor> indexAccessors;
@@ -48,6 +50,17 @@ public class LSMInvertedIndexSearchCursorInitialState implements ICursorInitialS
private final List<ILSMComponent> operationalComponents;
+ // For disk-based inverted list cursors
+ private int invListStartPageId = INVALID_VALUE;
+ private int invListEndPageId = INVALID_VALUE;
+ private int invListStartOffset = INVALID_VALUE;
+ private int invListNumElements = INVALID_VALUE;
+
+ public LSMInvertedIndexSearchCursorInitialState() {
+ this(null, null, null, null, null, null, false, null, null);
+ resetInvertedListInfo();
+ }
+
public LSMInvertedIndexSearchCursorInitialState(final MultiComparator keyCmp, PermutingTupleReference keysOnlyTuple,
List<IIndexAccessor> indexAccessors, List<IIndexAccessor> deletedKeysBTreeAccessors,
ITreeIndexFrameFactory deletedKeysBtreeLeafFrameFactory, IIndexOperationContext ctx,
@@ -61,7 +74,8 @@ public class LSMInvertedIndexSearchCursorInitialState implements ICursorInitialS
this.operationalComponents = operationalComponents;
this.lsmHarness = lsmHarness;
this.ctx = (LSMInvertedIndexOpContext) ctx;
- this.searchCallback = this.ctx.getSearchOperationCallback();
+ this.searchCallback = ctx != null ? this.ctx.getSearchOperationCallback() : null;
+ resetInvertedListInfo();
}
@Override
@@ -128,4 +142,35 @@ public class LSMInvertedIndexSearchCursorInitialState implements ICursorInitialS
public PermutingTupleReference getKeysOnlyTuple() {
return keysOnlyTuple;
}
+
+ public void setInvertedListInfo(int invListStartPageId, int invListEndPageId, int invListStartOffset,
+ int invListNumElements) {
+ this.invListStartPageId = invListStartPageId;
+ this.invListEndPageId = invListEndPageId;
+ this.invListStartOffset = invListStartOffset;
+ this.invListNumElements = invListNumElements;
+ }
+
+ public int getInvListStartPageId() {
+ return invListStartPageId;
+ }
+
+ public int getInvListEndPageId() {
+ return invListEndPageId;
+ }
+
+ public int getInvListStartOffset() {
+ return invListStartOffset;
+ }
+
+ public int getInvListNumElements() {
+ return invListNumElements;
+ }
+
+ private void resetInvertedListInfo() {
+ invListStartPageId = INVALID_VALUE;
+ invListEndPageId = INVALID_VALUE;
+ invListStartOffset = INVALID_VALUE;
+ invListNumElements = INVALID_VALUE;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
index 641ca3c..e74733b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java
@@ -18,11 +18,13 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.inmemory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import org.apache.hyracks.storage.am.btree.impls.BTree;
@@ -32,7 +34,7 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
@@ -150,29 +152,41 @@ public class InMemoryInvertedIndex implements IInPlaceInvertedIndex {
}
@Override
- public IInvertedListCursor createInvertedListCursor() {
+ public InvertedListCursor createInvertedListCursor(IHyracksTaskContext ctx) {
return new InMemoryInvertedListCursor(invListTypeTraits.length, tokenTypeTraits.length);
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey,
+ public InvertedListCursor createInvertedListRangeSearchCursor() {
+ // An in-memory index does not have a separate inverted list.
+ // Therefore, a different range-search cursor for an inverted list is not required.
+ return createInvertedListCursor(null);
+ }
+
+ @Override
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey,
IIndexOperationContext ictx) throws HyracksDataException {
InMemoryInvertedIndexOpContext ctx = (InMemoryInvertedIndexOpContext) ictx;
ctx.setOperation(IndexOperation.SEARCH);
InMemoryInvertedListCursor inMemListCursor = (InMemoryInvertedListCursor) listCursor;
inMemListCursor.prepare(ctx.getBtreeAccessor(), ctx.getBtreePred(), ctx.getTokenFieldsCmp(), ctx.getBtreeCmp());
inMemListCursor.reset(searchKey);
+ // Makes the cursor state to OPENED
+ inMemListCursor.open(null, null);
}
@Override
public InMemoryInvertedIndexAccessor createAccessor(IIndexAccessParameters iap) throws HyracksDataException {
return new InMemoryInvertedIndexAccessor(this,
- new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory));
+ new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory),
+ (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
- public InMemoryInvertedIndexAccessor createAccessor(int[] nonIndexFields) throws HyracksDataException {
+ public InMemoryInvertedIndexAccessor createAccessor(IIndexAccessParameters iap, int[] nonIndexFields)
+ throws HyracksDataException {
return new InMemoryInvertedIndexAccessor(this,
- new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory), nonIndexFields);
+ new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory), nonIndexFields,
+ (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
@Override
@@ -219,4 +233,5 @@ public class InMemoryInvertedIndex implements IInPlaceInvertedIndex {
public void purge() throws HyracksDataException {
btree.purge();
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
index 0795a4e..2a35301 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndexAccessor.java
@@ -19,7 +19,7 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.inmemory;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
@@ -31,8 +31,7 @@ import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex.DefaultHyracksCommonContext;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.TOccurrenceSearcher;
@@ -40,27 +39,29 @@ import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
public class InMemoryInvertedIndexAccessor implements IInvertedIndexAccessor {
- // TODO: This ctx needs to go away.
- protected final IHyracksCommonContext hyracksCtx = new DefaultHyracksCommonContext();
- protected final IInvertedIndexSearcher searcher;
+ protected final IHyracksTaskContext ctx;
+ protected IInvertedIndexSearcher searcher;
protected IIndexOperationContext opCtx;
protected InMemoryInvertedIndex index;
protected BTreeAccessor btreeAccessor;
private boolean destroyed = false;
- public InMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx)
- throws HyracksDataException {
+ public InMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx,
+ IHyracksTaskContext ctx) throws HyracksDataException {
+ this.ctx = ctx;
this.opCtx = opCtx;
this.index = index;
- this.searcher = createSearcher();
+ // Searcher will be initialized when conducting an actual search.
+ this.searcher = null;
this.btreeAccessor = index.getBTree().createAccessor(NoOpIndexAccessParameters.INSTANCE);
}
public InMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx,
- int[] nonIndexFields) throws HyracksDataException {
+ int[] nonIndexFields, IHyracksTaskContext ctx) throws HyracksDataException {
+ this.ctx = ctx;
this.opCtx = opCtx;
this.index = index;
- this.searcher = createSearcher();
+ this.searcher = null;
this.btreeAccessor = index.getBTree().createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE, nonIndexFields);
}
@@ -78,22 +79,28 @@ public class InMemoryInvertedIndexAccessor implements IInvertedIndexAccessor {
}
@Override
- public IIndexCursor createSearchCursor(boolean exclusive) {
- return new OnDiskInvertedIndexSearchCursor(searcher, index.getInvListTypeTraits().length);
+ public IIndexCursor createSearchCursor(boolean exclusive) throws HyracksDataException {
+ if (searcher == null) {
+ searcher = createSearcher();
+ }
+ return new OnDiskInvertedIndexSearchCursor(searcher);
}
@Override
public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException {
- searcher.search((OnDiskInvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
+ if (searcher == null) {
+ searcher = createSearcher();
+ }
+ searcher.search(cursor, (InvertedIndexSearchPredicate) searchPred, opCtx);
}
@Override
- public IInvertedListCursor createInvertedListCursor() {
- return index.createInvertedListCursor();
+ public InvertedListCursor createInvertedListCursor() {
+ return index.createInvertedListCursor(ctx);
}
@Override
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException {
index.openInvertedListCursor(listCursor, searchKey, opCtx);
}
@@ -124,7 +131,10 @@ public class InMemoryInvertedIndexAccessor implements IInvertedIndexAccessor {
}
protected IInvertedIndexSearcher createSearcher() throws HyracksDataException {
- return new TOccurrenceSearcher(hyracksCtx, index);
+ if (ctx != null) {
+ return new TOccurrenceSearcher(index, ctx);
+ }
+ return null;
}
public void resetLogTuple(ITupleReference newTuple) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index b2660a4..085f8d5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -33,11 +33,11 @@ import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
-public class InMemoryInvertedListCursor implements IInvertedListCursor {
+public class InMemoryInvertedListCursor extends InvertedListCursor {
private RangePredicate btreePred;
private BTreeAccessor btreeAccessor;
private IIndexCursor btreeCursor;
@@ -80,7 +80,7 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public int compareTo(IInvertedListCursor cursor) {
+ public int compareTo(InvertedListCursor cursor) {
try {
return size() - cursor.size();
} catch (HyracksDataException hde) {
@@ -100,12 +100,13 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public void reset(int startPageId, int endPageId, int startOff, int numElements) {
- // Do nothing
+ protected void setInvListInfo(int startPageId, int endPageId, int startOff, int numElements)
+ throws HyracksDataException {
+ // no-op for this in-memory cursor - everything is in memory
}
@Override
- public void pinPages() throws HyracksDataException {
+ public void loadPages() throws HyracksDataException {
btreePred.setLowKeyComparator(tokenFieldsCmp);
btreePred.setHighKeyComparator(tokenFieldsCmp);
btreePred.setLowKey(tokenTuple, true);
@@ -115,7 +116,7 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public void unpinPages() throws HyracksDataException {
+ public void unloadPages() throws HyracksDataException {
if (cursorNeedsClose) {
btreeCursor.close();
cursorNeedsClose = false;
@@ -123,17 +124,17 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
return btreeCursor.hasNext();
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
btreeCursor.next();
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
resultTuple.reset(btreeCursor.getTuple());
return resultTuple;
}
@@ -161,24 +162,9 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public int getStartPageId() {
- return 0;
- }
-
- @Override
- public int getEndPageId() {
- return 0;
- }
-
- @Override
- public int getStartOff() {
- return 0;
- }
-
- @Override
public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException {
// Close cursor if necessary.
- unpinPages();
+ unloadPages();
btreeSearchTuple.addTuple(searchTuple);
btreePred.setLowKeyComparator(btreeCmp);
btreePred.setHighKeyComparator(btreeCmp);
@@ -226,4 +212,20 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
return null;
}
+
+ @Override
+ public void prepareLoadPages() throws HyracksDataException {
+ // no-op for this in-memory cursor - no need to initialize a buffer
+ }
+
+ @Override
+ public void doClose() throws HyracksDataException {
+ btreeCursor.close();
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ btreeCursor.destroy();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
index 5ddba98..986ceac 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.inmemory;
-import java.util.List;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -31,7 +30,6 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IPartitionedInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedListPartitions;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher;
@@ -89,20 +87,21 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl
public PartitionedInMemoryInvertedIndexAccessor createAccessor(IIndexAccessParameters iap)
throws HyracksDataException {
return new PartitionedInMemoryInvertedIndexAccessor(this,
- new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory));
+ new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory), iap);
}
@Override
- public PartitionedInMemoryInvertedIndexAccessor createAccessor(int[] nonIndexFields) throws HyracksDataException {
+ public PartitionedInMemoryInvertedIndexAccessor createAccessor(IIndexAccessParameters iap, int[] nonIndexFields)
+ throws HyracksDataException {
return new PartitionedInMemoryInvertedIndexAccessor(this,
new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory),
- nonIndexFields);
+ nonIndexFields, iap);
}
@Override
public boolean openInvertedListPartitionCursors(IInvertedIndexSearcher searcher, IIndexOperationContext ictx,
- short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions,
- List<IInvertedListCursor> cursorsOrderedByTokens) throws HyracksDataException {
+ short numTokensLowerBound, short numTokensUpperBound, InvertedListPartitions invListPartitions)
+ throws HyracksDataException {
short minPartitionIndex;
short maxPartitionIndex;
partitionIndexLock.readLock().lock();
@@ -140,6 +139,8 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl
inMemListCursor.prepare(ctx.getBtreeAccessor(), ctx.getBtreePred(), ctx.getTokenFieldsCmp(),
ctx.getBtreeCmp());
inMemListCursor.reset(searchKey);
+ // Makes the cursor state to OPENED
+ inMemListCursor.open(null, null);
invListPartitions.addInvertedListCursor(inMemListCursor, i);
}
return true;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndexAccessor.java
index a4537a9..b5044d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndexAccessor.java
@@ -19,24 +19,28 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.inmemory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.PartitionedTOccurrenceSearcher;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
public class PartitionedInMemoryInvertedIndexAccessor extends InMemoryInvertedIndexAccessor {
- public PartitionedInMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx)
- throws HyracksDataException {
- super(index, opCtx);
+ public PartitionedInMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx,
+ IIndexAccessParameters iap) throws HyracksDataException {
+ super(index, opCtx, (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
public PartitionedInMemoryInvertedIndexAccessor(InMemoryInvertedIndex index, IIndexOperationContext opCtx,
- int[] nonIndexFields) throws HyracksDataException {
- super(index, opCtx, nonIndexFields);
+ int[] nonIndexFields, IIndexAccessParameters iap) throws HyracksDataException {
+ super(index, opCtx, nonIndexFields,
+ (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT));
}
protected IInvertedIndexSearcher createSearcher() throws HyracksDataException {
- return new PartitionedTOccurrenceSearcher(hyracksCtx, index);
+ return new PartitionedTOccurrenceSearcher(index, ctx);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index eec2993..da3f079 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -22,152 +22,351 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
+/**
+ * A cursor class that traverse an inverted list that consists of fixed-size elements on disk
+ *
+ */
+public class FixedSizeElementInvertedListCursor extends InvertedListCursor {
private final IBufferCache bufferCache;
private final int fileId;
private final int elementSize;
- private int currentElementIx;
- private int currentOff;
- private int currentPageIx;
-
+ // for sequential scan
+ private int currentElementIxForScan;
+ private int currentOffsetForScan;
+ private int currentPageIxForScan;
+ // the whole range of the given inverted list
private int startPageId;
private int endPageId;
private int startOff;
private int numElements;
+ private int numPages;
+ // the current range of the loaded pages in memory
+ private int bufferStartPageId;
+ private int bufferEndPageId;
+ private int bufferStartElementIx;
+ private int bufferEndElementIx;
+ private int bufferNumLoadedPages;
private final FixedSizeTupleReference tuple;
- private ICachedPage[] pages = new ICachedPage[10];
+ // The last element in the current range in memory
+ private final FixedSizeTupleReference bufferEndElementTuple;
+ private ICachedPage page;
+ // The last element index per page
private int[] elementIndexes = new int[10];
-
- private boolean pinned = false;
-
- public FixedSizeElementInvertedListCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields) {
+ // buffer manager to conform to the memory budget
+ private final ISimpleFrameBufferManager bufferManagerForSearch;
+ private ArrayList<ByteBuffer> buffers;
+ private boolean moreBlocksToRead = true;
+ // The last searched element index (used for random traversal)
+ private int lastRandomSearchedElementIx;
+
+ public FixedSizeElementInvertedListCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields,
+ IHyracksTaskContext ctx) throws HyracksDataException {
this.bufferCache = bufferCache;
this.fileId = fileId;
- this.currentElementIx = 0;
- this.currentPageIx = 0;
-
- int tmp = 0;
+ int tmpSize = 0;
for (int i = 0; i < invListFields.length; i++) {
- tmp += invListFields[i].getFixedLength();
+ tmpSize += invListFields[i].getFixedLength();
}
- elementSize = tmp;
- this.currentOff = -elementSize;
+ elementSize = tmpSize;
+ this.currentOffsetForScan = -elementSize;
+ this.currentElementIxForScan = 0;
+ this.currentPageIxForScan = 0;
+ this.bufferStartPageId = 0;
+ this.bufferEndPageId = 0;
+ this.bufferStartElementIx = 0;
+ this.bufferEndElementIx = 0;
+ this.bufferNumLoadedPages = 0;
+ this.lastRandomSearchedElementIx = 0;
+ this.moreBlocksToRead = true;
this.tuple = new FixedSizeTupleReference(invListFields);
+ this.bufferEndElementTuple = new FixedSizeTupleReference(invListFields);
+ this.buffers = new ArrayList<ByteBuffer>();
+ if (ctx == null) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_HYRACKS_TASK_IS_NULL);
+ }
+ this.bufferManagerForSearch = TaskUtil.get(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, ctx);
+ if (bufferManagerForSearch == null) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL);
+ }
+ }
+
+ /**
+ * Tries to allocate enough buffers to read the inverted list at once. If memory budget is not enough, this method
+ * stops allocating buffers.
+ */
+ private void allocateBuffers() throws HyracksDataException {
+ do {
+ ByteBuffer tmpBuffer = bufferManagerForSearch.acquireFrame(bufferCache.getPageSize());
+ if (tmpBuffer == null) {
+ // Budget exhausted
+ break;
+ }
+ Arrays.fill(tmpBuffer.array(), (byte) 0);
+ buffers.add(tmpBuffer);
+ } while (buffers.size() < numPages);
+ // At least there should be one frame to load a page from disk.
+ if (buffers.isEmpty()) {
+ throw HyracksDataException.create(ErrorCode.NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH,
+ FixedSizeElementInvertedListCursor.class.getName());
+ }
+ }
+
+ /**
+ * Deallocates all buffers. i.e. releases all buffers to the buffer manager.
+ */
+ private void deallocateBuffers() throws HyracksDataException {
+ for (int i = 0; i < buffers.size(); i++) {
+ bufferManagerForSearch.releaseFrame(buffers.get(i));
+ buffers.set(i, null);
+ }
+ buffers.clear();
}
+ /**
+ * Clears the contents of the buffers.
+ */
+ private void clearBuffers() throws HyracksDataException {
+ for (int i = 0; i < buffers.size(); i++) {
+ Arrays.fill(buffers.get(i).array(), (byte) 0);
+ buffers.get(i).clear();
+ }
+ }
+
+ /**
+ * Checks whether there are more elements to return. This is usually used for a sequential scan.
+ */
@Override
- public boolean hasNext() {
- return currentElementIx < numElements;
+ public boolean doHasNext() {
+ return currentElementIxForScan < numElements;
}
+ /**
+ * Returns the next element.
+ */
@Override
- public void next() {
- if (currentOff + 2 * elementSize > bufferCache.getPageSize()) {
- currentPageIx++;
- currentOff = 0;
+ public void doNext() throws HyracksDataException {
+ if (currentOffsetForScan + 2 * elementSize > bufferCache.getPageSize()) {
+ currentPageIxForScan++;
+ currentOffsetForScan = 0;
} else {
- currentOff += elementSize;
+ currentOffsetForScan += elementSize;
+ }
+
+ // Needs to read the next block?
+ if (currentElementIxForScan > bufferEndElementIx && endPageId > bufferEndPageId) {
+ loadPages();
+ currentOffsetForScan = 0;
+ }
+
+ currentElementIxForScan++;
+
+ tuple.reset(buffers.get(currentPageIxForScan).array(), currentOffsetForScan);
+ }
+
+ /**
+ * Prepares buffers to load pages. This method should not be called during the open()
+ * since it tries to allocate all available frames. If there are multiple concurrently opened
+ * cursors (e.g., a partitioned inverted index), this will cause an issue. An assumption of this cursor is
+ * that no two cursors are accessed at the same time even though they can be opened together.
+ */
+ @Override
+ public void prepareLoadPages() throws HyracksDataException {
+ // Resets the buffers if there is any.
+ clearBuffers();
+ if (numPages > buffers.size()) {
+ allocateBuffers();
}
- currentElementIx++;
- tuple.reset(pages[currentPageIx].getBuffer().array(), currentOff);
}
+ /**
+ * Reads a part of the inverted list into the working memory via the buffer cache.
+ * This method reads the inverted list until it fills the current buffers.
+ */
@Override
- public void pinPages() throws HyracksDataException {
- if (pinned) {
+ public void loadPages() throws HyracksDataException {
+ // Conducts a load. Based on the size of the buffers, it may be possible to read the entire list.
+ // Resets the start page ID to load. At this moment, the variable bufferEndPageId holds
+ // the last page ID where the previous loadPages() stopped.
+ bufferStartPageId = bufferEndPageId + 1;
+ if (bufferStartPageId > endPageId) {
return;
}
- int pix = 0;
- for (int i = startPageId; i <= endPageId; i++) {
- pages[pix] = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i), false);
- pix++;
+ int currentBufferIdx = 0;
+ ByteBuffer tmpBuffer;
+ for (int i = bufferStartPageId; i <= endPageId; i++) {
+ page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i), false);
+ // Copies the content to the buffer (working memory).
+ // Assumption: processing inverted list takes time; so, we don't want to keep them on the buffer cache.
+ // Rather, we utilize the assigned working memory (buffers).
+ tmpBuffer = page.getBuffer();
+ tmpBuffer.rewind();
+ buffers.get(currentBufferIdx).rewind();
+ buffers.get(currentBufferIdx).put(tmpBuffer);
+
+ currentBufferIdx++;
+ bufferCache.unpin(page);
+ bufferEndPageId = i;
+
+ // Buffer full?
+ if (currentBufferIdx >= buffers.size()) {
+ break;
+ }
}
- pinned = true;
+ setBlockInfo();
}
- @Override
- public void unpinPages() throws HyracksDataException {
- int numPages = endPageId - startPageId + 1;
- for (int i = 0; i < numPages; i++) {
- bufferCache.unpin(pages[i]);
+ /**
+ * Updates the information about this block.
+ */
+ private void setBlockInfo() {
+ bufferNumLoadedPages = bufferEndPageId - bufferStartPageId + 1;
+ bufferStartElementIx =
+ bufferStartPageId == startPageId ? 0 : elementIndexes[bufferStartPageId - startPageId - 1] + 1;
+ lastRandomSearchedElementIx = bufferStartElementIx;
+ bufferEndElementIx = elementIndexes[bufferEndPageId - startPageId];
+ // Gets the final element tuple in this block.
+ getElementAtIndex(bufferEndElementIx, bufferEndElementTuple);
+ currentPageIxForScan = 0;
+ currentOffsetForScan = bufferStartElementIx == 0 ? startOff - elementSize : -elementSize;
+ if (bufferEndPageId == endPageId) {
+ moreBlocksToRead = false;
}
- pinned = false;
}
- private void positionCursor(int elementIx) {
- int numPages = endPageId - startPageId + 1;
+ /**
+ * Unloads the pages from the buffers (working memory). This will release all buffers.
+ */
+ @Override
+ public void unloadPages() throws HyracksDataException {
+ // Deallocates the buffer pages
+ deallocateBuffers();
+ }
- currentPageIx = binarySearch(elementIndexes, 0, numPages, elementIx);
+ /**
+ * Checks whether the search tuple is greater than the last element in the current block of the cursor.
+ * If so, the cursor needs to load next block of the inverted list.
+ *
+ * @param searchTuple
+ * @param invListCmp
+ * @return true if the search tuple is greater than the last element in the current block of the cursor
+ * false if the search tuple is equal to or less than the last element in the current block of the cursor
+ * @throws HyracksDataException
+ */
+ private boolean needToReadNextBlock(ITupleReference searchTuple, MultiComparator invListCmp)
+ throws HyracksDataException {
+ if (moreBlocksToRead && invListCmp.compare(searchTuple, bufferEndElementTuple) > 0) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Gets the tuple for the given element index.
+ */
+ private void getElementAtIndex(int elementIx, FixedSizeTupleReference tuple) {
+ int currentPageIx =
+ binarySearch(elementIndexes, bufferStartPageId - startPageId, bufferNumLoadedPages, elementIx);
if (currentPageIx < 0) {
throw new IndexOutOfBoundsException(
"Requested index: " + elementIx + " from array with numElements: " + numElements);
}
+ int currentOff;
if (currentPageIx == 0) {
currentOff = startOff + elementIx * elementSize;
} else {
int relativeElementIx = elementIx - elementIndexes[currentPageIx - 1] - 1;
currentOff = relativeElementIx * elementSize;
}
-
- currentElementIx = elementIx;
- tuple.reset(pages[currentPageIx].getBuffer().array(), currentOff);
+ // Gets the actual index in the buffers since buffers.size() can be smaller than the total number of pages.
+ int bufferIdx = currentPageIx % buffers.size();
+ tuple.reset(buffers.get(bufferIdx).array(), currentOff);
}
+ /**
+ * Checks whether the given tuple exists on this inverted list. This method is used when doing a random traversal.
+ */
@Override
public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException {
- int mid;
- int begin = 0;
- int end = numElements - 1;
+ // If the given element is greater than the last element in the current buffer, reads the next block.
+ if (needToReadNextBlock(searchTuple, invListCmp)) {
+ loadPages();
+ }
+ int mid = -1;
+ int begin = lastRandomSearchedElementIx;
+ int end = bufferEndElementIx;
while (begin <= end) {
mid = (begin + end) / 2;
- positionCursor(mid);
+ getElementAtIndex(mid, tuple);
int cmp = invListCmp.compare(searchTuple, tuple);
if (cmp < 0) {
end = mid - 1;
} else if (cmp > 0) {
begin = mid + 1;
} else {
+ lastRandomSearchedElementIx = mid;
return true;
}
}
+ lastRandomSearchedElementIx = mid;
return false;
}
+ /**
+ * Opens the cursor for the given inverted list. After this open() call, prepreLoadPages() should be called
+ * before loadPages() are called. For more details, check prepapreLoadPages().
+ */
@Override
- public void reset(int startPageId, int endPageId, int startOff, int numElements) {
+ protected void setInvListInfo(int startPageId, int endPageId, int startOff, int numElements)
+ throws HyracksDataException {
this.startPageId = startPageId;
this.endPageId = endPageId;
this.startOff = startOff;
this.numElements = numElements;
- this.currentElementIx = 0;
- this.currentPageIx = 0;
- this.currentOff = startOff - elementSize;
-
- int numPages = endPageId - startPageId + 1;
- if (numPages > pages.length) {
- pages = new ICachedPage[endPageId - startPageId + 1];
- elementIndexes = new int[endPageId - startPageId + 1];
+ this.currentElementIxForScan = 0;
+ this.currentPageIxForScan = 0;
+ this.currentOffsetForScan = startOff - elementSize;
+ this.bufferStartPageId = startPageId;
+ // Deducts 1 since the startPage would be set to bufferEndPageId + 1 in loadPages().
+ this.bufferEndPageId = startPageId - 1;
+ this.moreBlocksToRead = true;
+ this.numPages = endPageId - startPageId + 1;
+
+ if (numPages > elementIndexes.length) {
+ elementIndexes = new int[numPages];
+ }
+
+ for (ByteBuffer buffer : buffers) {
+ buffer.clear();
}
- // fill elementIndexes
+ // Fills the last element index per page.
// first page
int cumulElements = (bufferCache.getPageSize() - startOff) / elementSize;
+ // Deducts 1 because this is the index, not the number of elements.
elementIndexes[0] = cumulElements - 1;
// middle, full pages
@@ -176,19 +375,23 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
}
// last page
+ // Deducts 1 because this is the index, not the number of elements.
elementIndexes[numPages - 1] = numElements - 1;
}
+ /**
+ * Prints the contents of the current inverted list (a debugging method).
+ */
@SuppressWarnings("rawtypes")
@Override
public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
- int oldCurrentOff = currentOff;
- int oldCurrentPageId = currentPageIx;
- int oldCurrentElementIx = currentElementIx;
+ int oldCurrentOff = currentOffsetForScan;
+ int oldCurrentPageId = currentPageIxForScan;
+ int oldCurrentElementIx = currentElementIxForScan;
- currentOff = startOff - elementSize;
- currentPageIx = 0;
- currentElementIx = 0;
+ currentOffsetForScan = startOff - elementSize;
+ currentPageIxForScan = 0;
+ currentElementIxForScan = 0;
StringBuilder strBuilder = new StringBuilder();
@@ -208,13 +411,16 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
}
// reset previous state
- currentOff = oldCurrentOff;
- currentPageIx = oldCurrentPageId;
- currentElementIx = oldCurrentElementIx;
+ currentOffsetForScan = oldCurrentOff;
+ currentPageIxForScan = oldCurrentPageId;
+ currentElementIxForScan = oldCurrentElementIx;
return strBuilder.toString();
}
+ /**
+ * Prints the current element (a debugging method).
+ */
@Override
@SuppressWarnings("rawtypes")
public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
@@ -232,6 +438,9 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
return strBuilder.toString();
}
+ /**
+ * Conducts a binary search to get the index of the given key.
+ */
private int binarySearch(int[] arr, int arrStart, int arrLength, int key) {
int mid;
int begin = arrStart;
@@ -259,8 +468,11 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
}
}
+ /**
+ * A compare function that is used to sort inverted list cursors
+ */
@Override
- public int compareTo(IInvertedListCursor invListCursor) {
+ public int compareTo(InvertedListCursor invListCursor) {
try {
return numElements - invListCursor.size();
} catch (HyracksDataException hde) {
@@ -268,36 +480,40 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
}
}
- @Override
- public int getEndPageId() {
- return endPageId;
- }
-
+ /**
+ * Gets the cardinality of the current inverted list.
+ */
@Override
public int size() {
return numElements;
}
+ /**
+ * Gets the current tuple.
+ */
@Override
- public int getStartOff() {
- return startOff;
+ public ITupleReference doGetTuple() {
+ return tuple;
}
+ /**
+ * Closes the cursor.
+ */
@Override
- public int getStartPageId() {
- return startPageId;
- }
-
- public int getOffset() {
- return currentOff;
- }
-
- public ICachedPage getPage() {
- return pages[currentPageIx];
+ public void doClose() throws HyracksDataException {
+ if (!buffers.isEmpty()) {
+ unloadPages();
+ }
}
+ /**
+ * Destroys the cursor.
+ */
@Override
- public ITupleReference getTuple() {
- return tuple;
+ public void doDestroy() throws HyracksDataException {
+ if (!buffers.isEmpty()) {
+ unloadPages();
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
new file mode 100644
index 0000000..ca0f40b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+/**
+ * A simple scan cursor that only reads a frame by frame from the inverted list. This cursor does not
+ * conduct a binary search. It only supports the scan operation. The main purpose of this cursor is
+ * doing a full-scan of an inverted list during a storage-component-merge process.
+ */
+public class FixedSizeElementInvertedListScanCursor extends InvertedListCursor {
+
+ protected final IBufferCache bufferCache;
+ protected final int fileId;
+ protected final int elementSize;
+ protected int currentElementIxForScan;
+ protected int currentOffsetForScan;
+ protected int currentPageId;
+
+ protected int startPageId;
+ protected int endPageId;
+ protected int startOff;
+ protected int numElements;
+ protected int numPages;
+
+ protected final FixedSizeTupleReference tuple;
+ protected ICachedPage page;
+
+ protected boolean pinned;
+
+ public FixedSizeElementInvertedListScanCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields)
+ throws HyracksDataException {
+ this.bufferCache = bufferCache;
+ this.fileId = fileId;
+ int tmpSize = 0;
+ for (int i = 0; i < invListFields.length; i++) {
+ tmpSize += invListFields[i].getFixedLength();
+ }
+ elementSize = tmpSize;
+ this.currentElementIxForScan = 0;
+ this.currentOffsetForScan = -elementSize;
+ this.currentPageId = 0;
+ this.startPageId = 0;
+ this.endPageId = 0;
+ this.startOff = 0;
+ this.numElements = 0;
+ this.numPages = 0;
+ this.tuple = new FixedSizeTupleReference(invListFields);
+ this.pinned = false;
+ }
+
+ @Override
+ public boolean doHasNext() {
+ return currentElementIxForScan < numElements;
+ }
+
+ @Override
+ public void doNext() throws HyracksDataException {
+ if (currentOffsetForScan + 2 * elementSize > bufferCache.getPageSize()) {
+ // Read the next page.
+ currentOffsetForScan = 0;
+ loadPages();
+ } else {
+ currentOffsetForScan += elementSize;
+ }
+ currentElementIxForScan++;
+ tuple.reset(page.getBuffer().array(), currentOffsetForScan);
+ }
+
+ @Override
+ public void prepareLoadPages() throws HyracksDataException {
+ // No-op for this cursor since it only loads one page to the buffer cache at a time.
+ }
+
+ /**
+ * Loads one page from the inverted list into the buffer cache.
+ */
+ @Override
+ public void loadPages() throws HyracksDataException {
+ if (pinned) {
+ unloadPages();
+ }
+ if (currentPageId == endPageId) {
+ return;
+ }
+ currentPageId++;
+ page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
+ page.acquireReadLatch();
+ pinned = true;
+ }
+
+ @Override
+ public void unloadPages() throws HyracksDataException {
+ if (pinned) {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ pinned = false;
+ }
+ }
+
+ @Override
+ protected void setInvListInfo(int startPageId, int endPageId, int startOff, int numElements)
+ throws HyracksDataException {
+ this.startPageId = startPageId;
+ this.endPageId = endPageId;
+ this.startOff = startOff;
+ this.numElements = numElements;
+ this.currentElementIxForScan = 0;
+ this.currentOffsetForScan = startOff - elementSize;
+ // Deducts 1 since the startPage would be set to bufferCurrentPageId + 1 in loadPages().
+ this.currentPageId = startPageId - 1;
+ this.numPages = endPageId - startPageId + 1;
+ this.pinned = false;
+ }
+
+ @Override
+ public int compareTo(InvertedListCursor invListCursor) {
+ try {
+ return numElements - invListCursor.size();
+ } catch (HyracksDataException hde) {
+ throw new IllegalStateException(hde);
+ }
+ }
+
+ @Override
+ public int size() {
+ return numElements;
+ }
+
+ @Override
+ public ITupleReference doGetTuple() {
+ return tuple;
+ }
+
+ @Override
+ public void doClose() throws HyracksDataException {
+ if (pinned) {
+ unloadPages();
+ }
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ if (pinned) {
+ unloadPages();
+ }
+ }
+
+ @Override
+ public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) throws HyracksDataException {
+ // This method is designed for a random search.
+ return false;
+ }
+
+ @Override
+ public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
+ return null;
+ }
+
+ @Override
+ public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
+ return null;
+ }
+
+}
[2/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListMerger.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListMerger.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListMerger.java
index 81b6467..0bfc140 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListMerger.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListMerger.java
@@ -20,281 +20,376 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.search;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.std.buffermanager.ISimpleFrameBufferManager;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAccessor;
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeTupleReference;
import org.apache.hyracks.storage.common.MultiComparator;
-// TODO: The merge procedure is rather confusing regarding cursor positions, hasNext() calls etc.
-// Needs an overhaul some time.
+/**
+ * Conducts the merging operation among all inverted list cursors and generates the final result.
+ */
public class InvertedListMerger {
- protected final MultiComparator invListCmp;
- protected SearchResult prevSearchResult;
- protected SearchResult newSearchResult;
+ // current merge process type
+ public enum processType {
+ PREFIX_LIST,
+ SUFFIX_LIST_PROBE,
+ SUFFIX_LIST_SCAN,
+ NONE
+ }
- public InvertedListMerger(IHyracksCommonContext ctx, IInvertedIndex invIndex) throws HyracksDataException {
+ protected final MultiComparator invListCmp;
+ protected InvertedIndexSearchResult prevSearchResult;
+ protected InvertedIndexSearchResult newSearchResult;
+ protected InvertedIndexFinalSearchResult finalSearchResult;
+
+ // To Keep the status of this merge process since we only calculate one frame at a time in case of the final result
+ protected InvertedListCursor finalInvListCursor;
+ protected int occurrenceThreshold;
+ protected int numInvertedLists;
+ protected int invListIdx;
+ protected int numPrevResult;
+ protected int prevBufIdx;
+ protected int maxPrevBufIdx;
+ protected int numExpectedPages;
+ protected ByteBuffer prevCurrentBuffer;
+ protected FixedSizeFrameTupleAccessor resultFrameTupleAcc;
+ protected FixedSizeTupleReference resultTuple;
+ protected boolean advanceCursor;
+ protected boolean advancePrevResult;
+ protected int resultTidx;
+ protected int invListTidx;
+ protected int invListTupleCount;
+ protected ITupleReference invListTuple;
+ protected int prevResultFrameTupleCount;
+
+ // Entire calculation done?
+ protected boolean isProcessingFinished;
+ // Dealing with the final list?
+ protected boolean isProcessingFinalList;
+ // Dealing with the final partition?
+ protected boolean isProcessingFinalPartition;
+ // Variable Initialization done?
+ protected boolean listVisited;
+ protected processType currentProcessType = processType.NONE;
+
+ public InvertedListMerger(IHyracksTaskContext ctx, IInvertedIndex invIndex, ISimpleFrameBufferManager bufferManager)
+ throws HyracksDataException {
this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
- this.prevSearchResult = new SearchResult(invIndex.getInvListTypeTraits(), ctx);
- this.newSearchResult = new SearchResult(prevSearchResult);
+ this.prevSearchResult = new InvertedIndexSearchResult(invIndex.getInvListTypeTraits(), ctx, bufferManager);
+ this.newSearchResult = new InvertedIndexSearchResult(invIndex.getInvListTypeTraits(), ctx, bufferManager);
}
- public void merge(ArrayList<IInvertedListCursor> invListCursors, int occurrenceThreshold, int numPrefixLists,
- SearchResult searchResult) throws HyracksDataException {
+ /**
+ * Generates the result based on the given occurrenceThreshold. For the prefix lists, we do merge.
+ * For the suffix lists, we either conduct a binary search or a scan for each record ID.
+ *
+ * @return true only if all processing for the final list for a partition is done.
+ * false otherwise.
+ * @throws HyracksDataException
+ */
+ public boolean merge(List<InvertedListCursor> invListCursors, int occurrenceThreshold, int numPrefixLists,
+ InvertedIndexFinalSearchResult finalSearchResult) throws HyracksDataException {
Collections.sort(invListCursors);
int numInvLists = invListCursors.size();
- SearchResult result = null;
+ InvertedIndexSearchResult result = null;
+ prevSearchResult.reset();
+ newSearchResult.reset();
+ boolean isFinalList = false;
+ // This will be only set to true when the processing the last list in a partition is done.
+ boolean doneMerge = false;
+ this.occurrenceThreshold = occurrenceThreshold;
for (int i = 0; i < numInvLists; i++) {
- SearchResult swapTemp = prevSearchResult;
+ // Sets the previous search result and the new (current) search result that will be written.
+ InvertedIndexSearchResult swapTemp = prevSearchResult;
prevSearchResult = newSearchResult;
newSearchResult = swapTemp;
newSearchResult.reset();
if (i + 1 != numInvLists) {
- // Use temporary search results when not merging last list.
+ // Use a temporary intermediate search result when not merging last list.
result = newSearchResult;
} else {
- // When merging the last list, append results to the final search result.
- result = searchResult;
+ // When merging the last list, appends results to the final search result. This is needed since
+ // this merge() can be called multiple times in case of partitioned T-occurrrence searches.
+ // So, we need to keep a separate search result.
+ result = finalSearchResult;
+ isFinalList = true;
}
- IInvertedListCursor invListCursor = invListCursors.get(i);
- invListCursor.pinPages();
+ InvertedListCursor invListCursor = invListCursors.get(i);
+ // Loads the inverted list (at least some part of it).
+ invListCursor.prepareLoadPages();
+ invListCursor.loadPages();
if (i < numPrefixLists) {
- // Merge prefix list.
- mergePrefixList(invListCursor, prevSearchResult, result);
+ // Merges a prefix list.
+ doneMerge = mergePrefixList(invListCursor, prevSearchResult, result, isFinalList);
} else {
// Merge suffix list.
int numInvListElements = invListCursor.size();
int currentNumResults = prevSearchResult.getNumResults();
// Should we binary search the next list or should we sort-merge it?
if (currentNumResults * Math.log(numInvListElements) < currentNumResults + numInvListElements) {
- mergeSuffixListProbe(invListCursor, prevSearchResult, result, i, numInvLists, occurrenceThreshold);
+ doneMerge = mergeSuffixListProbe(invListCursor, prevSearchResult, result, i, numInvLists,
+ occurrenceThreshold, isFinalList);
} else {
- mergeSuffixListScan(invListCursor, prevSearchResult, result, i, numInvLists, occurrenceThreshold);
+ doneMerge = mergeSuffixListScan(invListCursor, prevSearchResult, result, i, numInvLists,
+ occurrenceThreshold, isFinalList);
+ }
+ }
+
+ if (isFinalList) {
+ // For the final list, the method unloadPages() should be called only when traversing
+ // the inverted list is finished.
+ if (doneMerge) {
+ invListCursor.unloadPages();
+ invListCursor.close();
}
+ // Needs to return the calculation result for the final list only.
+ // Otherwise, the process needs to be continued until this method traverses the final inverted list
+ // and either generates some output in the output buffer or finishes traversing it.
+ return doneMerge;
}
- invListCursor.unpinPages();
+
+ invListCursor.unloadPages();
+ invListCursor.close();
}
- }
- protected void mergeSuffixListProbe(IInvertedListCursor invListCursor, SearchResult prevSearchResult,
- SearchResult newSearchResult, int invListIx, int numInvLists, int occurrenceThreshold)
- throws HyracksDataException {
+ // Control does not reach here.
+ return true;
+ }
- int prevBufIdx = 0;
- int maxPrevBufIdx = prevSearchResult.getCurrentBufferIndex();
- ByteBuffer prevCurrentBuffer = prevSearchResult.getBuffers().get(0);
+ /**
+ * Continues the merge process on a final list if it has been paused because
+ * the output buffer of the final result was full.
+ *
+ * @return true only if all processing for the final list for a partition is done.
+ * false otherwise (still more to proceed).
+ */
+ public boolean continueMerge() throws HyracksDataException {
+ boolean doneMerge;
+ switch (currentProcessType) {
+ case PREFIX_LIST:
+ doneMerge =
+ mergePrefixList(finalInvListCursor, prevSearchResult, finalSearchResult, isProcessingFinalList);
+ break;
+ case SUFFIX_LIST_PROBE:
+ doneMerge = mergeSuffixListProbe(finalInvListCursor, prevSearchResult, finalSearchResult, invListIdx,
+ numInvertedLists, occurrenceThreshold, isProcessingFinalList);
+ break;
+ case SUFFIX_LIST_SCAN:
+ doneMerge = mergeSuffixListScan(finalInvListCursor, prevSearchResult, finalSearchResult, invListIdx,
+ numInvertedLists, occurrenceThreshold, isProcessingFinalList);
+ break;
+ default:
+ throw HyracksDataException.create(ErrorCode.UNDEFINED_INVERTED_LIST_MERGE_TYPE);
+ }
- FixedSizeFrameTupleAccessor resultFrameTupleAcc = prevSearchResult.getAccessor();
- FixedSizeTupleReference resultTuple = prevSearchResult.getTuple();
+ if (doneMerge) {
+ // Final calculation is done.
+ finalInvListCursor.unloadPages();
+ finalInvListCursor.close();
+ }
- int resultTidx = 0;
+ return doneMerge;
+ }
- resultFrameTupleAcc.reset(prevCurrentBuffer);
+ /**
+ * Merges the given suffix list to the previous result using the given inverted list cursor. When traversing the
+ * inverted list cursor, a binary search will be conducted for each tuple in the previous search result.
+ *
+ * @return true only if all processing for the final list for a partition is done.
+ * false otherwise.
+ */
+ protected boolean mergeSuffixListProbe(InvertedListCursor invListCursor, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, int invListIx, int numInvLists, int occurrenceThreshold,
+ boolean isFinalList) throws HyracksDataException {
+ if (isProcessingFinished) {
+ return true;
+ }
- while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
+ initMergingOneList(invListCursor, prevSearchResult, newSearchResult, isFinalList, invListIx, numInvLists,
+ occurrenceThreshold, processType.SUFFIX_LIST_PROBE);
+ while (resultTidx < prevResultFrameTupleCount) {
resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
-
+ int count = getCount(resultTuple);
if (invListCursor.containsKey(resultTuple, invListCmp)) {
+ // Found the same tuple again on the current list. Increases the count by one.
count++;
- newSearchResult.append(resultTuple, count);
- } else {
- if (count + numInvLists - invListIx > occurrenceThreshold) {
- newSearchResult.append(resultTuple, count);
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller consume the frame.
+ // SearchResult.append() should only return false for this case.
+ return false;
}
- }
-
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevSearchResult.getBuffers().get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
+ } else if (count + numInvLists - invListIx > occurrenceThreshold) {
+ // This tuple only exists on the previous result. However, it can be a part of the answer
+ // if it will be found again in the remaining lists.
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller consume the frame.
+ // SearchResult.append() should only return false for this case.
+ return false;
}
}
+ resultTidx++;
+ checkPrevResultAndFetchNextFrame(prevSearchResult);
}
- }
-
- protected void mergeSuffixListScan(IInvertedListCursor invListCursor, SearchResult prevSearchResult,
- SearchResult newSearchResult, int invListIx, int numInvLists, int occurrenceThreshold)
- throws HyracksDataException {
-
- int prevBufIdx = 0;
- int maxPrevBufIdx = prevSearchResult.getCurrentBufferIndex();
- ByteBuffer prevCurrentBuffer = prevSearchResult.getBuffers().get(0);
-
- FixedSizeFrameTupleAccessor resultFrameTupleAcc = prevSearchResult.getAccessor();
- FixedSizeTupleReference resultTuple = prevSearchResult.getTuple();
- boolean advanceCursor = true;
- boolean advancePrevResult = false;
- int resultTidx = 0;
-
- resultFrameTupleAcc.reset(prevCurrentBuffer);
-
- int invListTidx = 0;
- int invListNumTuples = invListCursor.size();
+ return finishMergingOneList(isFinalList, prevSearchResult, newSearchResult, invListCursor);
+ }
- if (invListCursor.hasNext()) {
- invListCursor.next();
+ /**
+ * Merges the given suffix list to the previous result using the given inverted list cursor. When traversing the
+ * inverted list cursor, a scan will be conducted like the mergePrefixList() method.
+ *
+ * @return true only if all processing for the final list for a partition is done.
+ * false otherwise.
+ */
+ protected boolean mergeSuffixListScan(InvertedListCursor invListCursor, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, int invListIx, int numInvLists, int occurrenceThreshold,
+ boolean isFinalList) throws HyracksDataException {
+ if (isProcessingFinished) {
+ return true;
}
- while (invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- ITupleReference invListTuple = invListCursor.getTuple();
+ // When generating the final result, we need to initialize the variable only once.
+ initMergingOneList(invListCursor, prevSearchResult, newSearchResult, isFinalList, invListIx, numInvLists,
+ occurrenceThreshold, processType.SUFFIX_LIST_SCAN);
+ int cmp;
+ int count;
+ while (invListTidx < invListTupleCount && resultTidx < prevResultFrameTupleCount) {
+ invListTuple = invListCursor.getTuple();
resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
-
- int cmp = invListCmp.compare(invListTuple, resultTuple);
+ cmp = invListCmp.compare(invListTuple, resultTuple);
if (cmp == 0) {
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1)) + 1;
- newSearchResult.append(resultTuple, count);
+ // Found the same tuple again on the current list. Increases the count by one.
+ // Also, both the result and the cursor advances.
+ count = getCount(resultTuple) + 1;
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller consume the frame.
+ // SearchResult.append() should only return false for this case.
+ return false;
+ }
advanceCursor = true;
advancePrevResult = true;
+ } else if (cmp < 0) {
+ // Found a new tuple on the current list. Based on prefix/suffix algorithm,
+ // this tuple can be ignored since it can't be a part of the answer.
+ advanceCursor = true;
+ advancePrevResult = false;
} else {
- if (cmp < 0) {
- advanceCursor = true;
- advancePrevResult = false;
- } else {
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
- if (count + numInvLists - invListIx > occurrenceThreshold) {
- newSearchResult.append(resultTuple, count);
- }
- advanceCursor = false;
- advancePrevResult = true;
- }
- }
-
- if (advancePrevResult) {
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevSearchResult.getBuffers().get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
+ // This tuple only exists on the previous result. However, it can be a part of the answer
+ // if it will be found again in the remaining lists.
+ count = getCount(resultTuple);
+ if (count + numInvLists - invListIx > occurrenceThreshold) {
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
}
}
+ advanceCursor = false;
+ advancePrevResult = true;
}
-
- if (advanceCursor) {
- invListTidx++;
- if (invListCursor.hasNext()) {
- invListCursor.next();
- }
- }
+ // Gets the next tuple from the previous result and the list cursor.
+ advancePrevResultAndList(advancePrevResult, advanceCursor, prevSearchResult, invListCursor);
}
// append remaining elements from previous result set
- while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
+ // These remaining elements can be a part of the answer if they will be found again in the remaining lists.
+ while (resultTidx < prevResultFrameTupleCount) {
resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
-
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
+ count = getCount(resultTuple);
if (count + numInvLists - invListIx > occurrenceThreshold) {
- newSearchResult.append(resultTuple, count);
- }
-
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevSearchResult.getBuffers().get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
}
}
+ resultTidx++;
+ checkPrevResultAndFetchNextFrame(prevSearchResult);
}
- }
-
- protected void mergePrefixList(IInvertedListCursor invListCursor, SearchResult prevSearchResult,
- SearchResult newSearchResult) throws HyracksDataException {
-
- int prevBufIdx = 0;
- int maxPrevBufIdx = prevSearchResult.getCurrentBufferIndex();
- ByteBuffer prevCurrentBuffer = prevSearchResult.getBuffers().get(0);
-
- FixedSizeFrameTupleAccessor resultFrameTupleAcc = prevSearchResult.getAccessor();
- FixedSizeTupleReference resultTuple = prevSearchResult.getTuple();
- boolean advanceCursor = true;
- boolean advancePrevResult = false;
- int resultTidx = 0;
-
- resultFrameTupleAcc.reset(prevCurrentBuffer);
-
- int invListTidx = 0;
- int invListNumTuples = invListCursor.size();
+ return finishMergingOneList(isFinalList, prevSearchResult, newSearchResult, invListCursor);
+ }
- if (invListCursor.hasNext()) {
- invListCursor.next();
+ /**
+ * Merges the prefix lists one by one. It reads the previous search result and one inverted list,
+ * then generates a new result by applying UNIONALL operation on these two. This method returns true
+ * only if all processing for the given final list is done. Otherwise, it returns false.
+ */
+ protected boolean mergePrefixList(InvertedListCursor invListCursor, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, boolean isFinalList) throws HyracksDataException {
+ if (isProcessingFinished) {
+ return true;
}
- while (invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
+ // Assigns necessary variables and fetches a tuple from the inverted list cursor.
+ initMergingOneList(invListCursor, prevSearchResult, newSearchResult, isFinalList, processType.PREFIX_LIST);
- ITupleReference invListTuple = invListCursor.getTuple();
+ int cmp;
+ int count;
+ // Traverses the inverted list and the previous result at the same time.
+ while (invListTidx < invListTupleCount && resultTidx < prevResultFrameTupleCount) {
+ invListTuple = invListCursor.getTuple();
resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
-
- int cmp = invListCmp.compare(invListTuple, resultTuple);
+ cmp = invListCmp.compare(invListTuple, resultTuple);
+ // Found the same tuple again on the current list: count + 1. Both the result and the cursor advances.
if (cmp == 0) {
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1)) + 1;
- newSearchResult.append(resultTuple, count);
+ count = getCount(resultTuple) + 1;
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
+ }
advanceCursor = true;
advancePrevResult = true;
- } else {
- if (cmp < 0) {
- int count = 1;
- newSearchResult.append(invListTuple, count);
- advanceCursor = true;
- advancePrevResult = false;
- } else {
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
- newSearchResult.append(resultTuple, count);
- advanceCursor = false;
- advancePrevResult = true;
+ } else if (cmp < 0) {
+ // Found a new tuple on the current list. Sets its count to 1. Only advances the cursor.
+ count = 1;
+ if (!newSearchResult.append(invListTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
}
- }
-
- if (advancePrevResult) {
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevSearchResult.getBuffers().get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
- }
- }
- }
-
- if (advanceCursor) {
- invListTidx++;
- if (invListCursor.hasNext()) {
- invListCursor.next();
+ advanceCursor = true;
+ advancePrevResult = false;
+ //
+ } else {
+ // This tuple only exists on the previous result. Maintains its count. Only advances the result.
+ count = getCount(resultTuple);
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
}
+ advanceCursor = false;
+ advancePrevResult = true;
+ //
}
+ // Gets the next tuple from the previous result and the list cursor.
+ advancePrevResultAndList(advancePrevResult, advanceCursor, prevSearchResult, invListCursor);
}
// append remaining new elements from inverted list
- while (invListTidx < invListNumTuples) {
- ITupleReference invListTuple = invListCursor.getTuple();
- newSearchResult.append(invListTuple, 1);
+ //
+ while (invListTidx < invListTupleCount) {
+ invListTuple = invListCursor.getTuple();
+ if (!newSearchResult.append(invListTuple, 1)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
+ }
invListTidx++;
if (invListCursor.hasNext()) {
invListCursor.next();
@@ -302,32 +397,188 @@ public class InvertedListMerger {
}
// append remaining elements from previous result set
- while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
+ while (resultTidx < prevResultFrameTupleCount) {
resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ count = getCount(resultTuple);
+ if (!newSearchResult.append(resultTuple, count)) {
+ // For a final result, needs to pause when a frame becomes full to let the caller
+ // consume the frame. SearchResult.append() should only return false for this case.
+ return false;
+ }
+ resultTidx++;
+ checkPrevResultAndFetchNextFrame(prevSearchResult);
+ }
+
+ return finishMergingOneList(isFinalList, prevSearchResult, newSearchResult, invListCursor);
+ }
+
+ /**
+ * Initializes necessary information for each merging operation (prefix_list) for a list.
+ */
+ protected void initMergingOneList(InvertedListCursor invListCursor, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, boolean isFinalList, processType mergeOpType)
+ throws HyracksDataException {
+ initMergingOneList(invListCursor, prevSearchResult, newSearchResult, isFinalList, 0, 0, 0, mergeOpType);
+ }
+
+ /**
+ * Initializes necessary information for each merging operation (suffix_list_probe or suffix_list_scan) for a list.
+ */
+ protected void initMergingOneList(InvertedListCursor invListCursor, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, boolean isFinalList, int invListIx, int numInvLists,
+ int occurrenceThreshold, processType mergeOpType) throws HyracksDataException {
+ // Each inverted list will be visited only once except the final inverted list.
+ // When generating the final result, the given inverted list can be visited multiple times
+ // since we only generate one frame at a time. So, we need to initialize the following variables only once.
+ if (!listVisited) {
+ resetVariable();
+ prevBufIdx = 0;
+ maxPrevBufIdx = prevSearchResult.getCurrentBufferIndex();
+ // Gets the maximum possible number of expected result pages (in case no common element at all).
+ numPrevResult = prevSearchResult.getNumResults();
+ invListTupleCount = invListCursor.size();
+ numExpectedPages = newSearchResult.getExpectedNumPages(numPrevResult + invListTupleCount);
+ newSearchResult.prepareWrite(numExpectedPages);
+ prevSearchResult.prepareResultRead();
+ prevCurrentBuffer = prevSearchResult.getNextFrame();
+ resultFrameTupleAcc = prevSearchResult.getAccessor();
+ resultTuple = prevSearchResult.getTuple();
+ advanceCursor = true;
+ advancePrevResult = false;
+ resultTidx = 0;
+ resultFrameTupleAcc.reset(prevCurrentBuffer);
+ invListTidx = 0;
+ numInvertedLists = numInvLists;
+ invListIdx = invListIx;
+ prevResultFrameTupleCount = prevCurrentBuffer == null ? 0 : resultFrameTupleAcc.getTupleCount();
+
+ // Additional variables to keep the current status of the given merge process
+ if (isFinalList) {
+ finalInvListCursor = invListCursor;
+ finalSearchResult = (InvertedIndexFinalSearchResult) newSearchResult;
+ currentProcessType = mergeOpType;
+ this.occurrenceThreshold = occurrenceThreshold;
+ isProcessingFinalList = true;
+ listVisited = true;
+ }
+
+ if (invListCursor.hasNext()) {
+ invListCursor.next();
+ }
+ }
+ }
- int count = IntegerPointable.getInteger(resultTuple.getFieldData(0),
- resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
- newSearchResult.append(resultTuple, count);
+ /**
+ * Finishes the merging operation of one list.
+ *
+ * @return true only if this merging operation is for a final list
+ * false otherwise
+ */
+ protected boolean finishMergingOneList(boolean isFinalList, InvertedIndexSearchResult prevSearchResult,
+ InvertedIndexSearchResult newSearchResult, InvertedListCursor invListCursor) throws HyracksDataException {
+ prevSearchResult.closeResultRead(false);
+ invListCursor.close();
+ // Final search result can be called multiple times for partitioned occurrence searcher case
+ // so that it can be written multiple times. So, should not finalize the write here.
+ // The caller of merge() should ensure that.
+ if (!isFinalList) {
+ newSearchResult.finalizeWrite();
+ return false;
+ } else {
+ // Final list? then, the calculation is done.
+ isProcessingFinished = true;
+ return true;
+ }
+ }
+ /**
+ * Fetches the next previous result frame if the current frame has been consumed.
+ * Also fetches next element from the inverted list cursor.
+ */
+ protected void advancePrevResultAndList(boolean advancePrevResult, boolean advanceCursor,
+ InvertedIndexSearchResult prevSearchResult, InvertedListCursor invListCursor) throws HyracksDataException {
+ if (advancePrevResult) {
resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevSearchResult.getBuffers().get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
- }
+ checkPrevResultAndFetchNextFrame(prevSearchResult);
+ }
+
+ if (advanceCursor) {
+ invListTidx++;
+ if (invListCursor.hasNext()) {
+ invListCursor.next();
}
}
}
- public SearchResult createSearchResult() throws HyracksDataException {
- return new SearchResult(prevSearchResult);
+ /**
+ * Fetches the next page of the previous result if possible.
+ */
+ protected void checkPrevResultAndFetchNextFrame(InvertedIndexSearchResult prevSearchResult)
+ throws HyracksDataException {
+ if (resultTidx >= prevResultFrameTupleCount) {
+ prevBufIdx++;
+ if (prevBufIdx <= maxPrevBufIdx) {
+ prevCurrentBuffer = prevSearchResult.getNextFrame();
+ resultFrameTupleAcc.reset(prevCurrentBuffer);
+ prevResultFrameTupleCount = resultFrameTupleAcc.getTupleCount();
+ resultTidx = 0;
+ }
+ }
}
- public void reset() {
- prevSearchResult.clear();
- newSearchResult.clear();
+ /**
+ * Gets the count of the given tuple in the previous search result.
+ */
+ protected int getCount(FixedSizeTupleReference resultTuple) {
+ return IntegerPointable.getInteger(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
}
+
+ public void reset() throws HyracksDataException {
+ prevSearchResult.reset();
+ newSearchResult.reset();
+ resetVariable();
+ }
+
+ /**
+ * Prepares the merge process. This mainly allocates buffers for the two intermediate search results.
+ */
+ public void prepareMerge() throws HyracksDataException {
+ prevSearchResult.prepareIOBuffer();
+ newSearchResult.prepareIOBuffer();
+ resetVariable();
+ }
+
+ /**
+ * Cleans every stuff.
+ */
+ public void close() throws HyracksDataException {
+ prevSearchResult.close();
+ newSearchResult.close();
+ }
+
+ // Resets the variables.
+ private void resetVariable() {
+ prevBufIdx = 0;
+ maxPrevBufIdx = 0;
+ numPrevResult = 0;
+ invListTupleCount = 0;
+ numExpectedPages = 0;
+ prevCurrentBuffer = null;
+ resultFrameTupleAcc = null;
+ resultTuple = null;
+ advanceCursor = false;
+ advancePrevResult = false;
+ resultTidx = 0;
+ invListTidx = 0;
+ prevResultFrameTupleCount = 0;
+ finalInvListCursor = null;
+ finalSearchResult = null;
+ currentProcessType = processType.NONE;
+ isProcessingFinalList = false;
+ isProcessingFinished = false;
+ listVisited = false;
+ occurrenceThreshold = 0;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListPartitions.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListPartitions.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListPartitions.java
index e9d6c1f..fef4baf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListPartitions.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/InvertedListPartitions.java
@@ -22,24 +22,29 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.search;
import java.util.ArrayList;
import java.util.Arrays;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IObjectFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.ObjectCache;
+/**
+ * This class contains multiple inverted list partitions. Each partition can contain inverted list cursors in it.
+ *
+ */
public class InvertedListPartitions {
private final int DEFAULT_NUM_PARTITIONS = 10;
private final int PARTITIONS_SLACK_SIZE = 10;
private final int OBJECT_CACHE_INIT_SIZE = 10;
private final int OBJECT_CACHE_EXPAND_SIZE = 10;
- private final IObjectFactory<ArrayList<IInvertedListCursor>> arrayListFactory;
- private final ObjectCache<ArrayList<IInvertedListCursor>> arrayListCache;
- private ArrayList<IInvertedListCursor>[] partitions;
+ private final IObjectFactory<ArrayList<InvertedListCursor>> arrayListFactory;
+ private final ObjectCache<ArrayList<InvertedListCursor>> arrayListCache;
+ private ArrayList<InvertedListCursor>[] partitions;
private short minValidPartitionIndex;
private short maxValidPartitionIndex;
- public InvertedListPartitions() {
- this.arrayListFactory = new ArrayListFactory<IInvertedListCursor>();
- this.arrayListCache = new ObjectCache<ArrayList<IInvertedListCursor>>(arrayListFactory, OBJECT_CACHE_INIT_SIZE,
+ public InvertedListPartitions() throws HyracksDataException {
+ this.arrayListFactory = new ArrayListFactory<InvertedListCursor>();
+ this.arrayListCache = new ObjectCache<ArrayList<InvertedListCursor>>(arrayListFactory, OBJECT_CACHE_INIT_SIZE,
OBJECT_CACHE_EXPAND_SIZE);
}
@@ -52,7 +57,7 @@ public class InvertedListPartitions {
} else {
initialSize = numTokensUpperBound + 1;
}
- partitions = (ArrayList<IInvertedListCursor>[]) new ArrayList[initialSize];
+ partitions = (ArrayList<InvertedListCursor>[]) new ArrayList[initialSize];
} else {
if (numTokensUpperBound + 1 >= partitions.length) {
partitions = Arrays.copyOf(partitions, numTokensUpperBound + 1);
@@ -64,11 +69,11 @@ public class InvertedListPartitions {
maxValidPartitionIndex = Short.MIN_VALUE;
}
- public void addInvertedListCursor(IInvertedListCursor listCursor, short numTokens) {
+ public void addInvertedListCursor(InvertedListCursor listCursor, short numTokens) throws HyracksDataException {
if (numTokens + 1 >= partitions.length) {
partitions = Arrays.copyOf(partitions, numTokens + PARTITIONS_SLACK_SIZE);
}
- ArrayList<IInvertedListCursor> partitionCursors = partitions[numTokens];
+ ArrayList<InvertedListCursor> partitionCursors = partitions[numTokens];
if (partitionCursors == null) {
partitionCursors = arrayListCache.getNext();
partitionCursors.clear();
@@ -84,7 +89,7 @@ public class InvertedListPartitions {
partitionCursors.add(listCursor);
}
- public ArrayList<IInvertedListCursor>[] getPartitions() {
+ public ArrayList<InvertedListCursor>[] getPartitions() {
return partitions;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
index 5ae4e05..3d8c35a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
@@ -20,9 +20,9 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.search;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.primitive.ShortPointable;
@@ -33,10 +33,13 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IPartitionedInvertedIndex;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexSearchCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
+import org.apache.hyracks.storage.common.IIndexCursor;
+/**
+ * Conducts T-Occurrence searches on inverted lists in one or more partition.
+ */
public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher {
protected final ArrayTupleBuilder lowerBoundTupleBuilder = new ArrayTupleBuilder(1);
@@ -45,17 +48,26 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
protected final ArrayTupleReference upperBoundTuple = new ArrayTupleReference();
protected final ConcatenatingTupleReference fullLowSearchKey = new ConcatenatingTupleReference(2);
protected final ConcatenatingTupleReference fullHighSearchKey = new ConcatenatingTupleReference(2);
-
- // Inverted list cursors ordered by token. Used to read relevant inverted-list partitions of one token one after
- // the other for better I/O performance (because the partitions of one inverted list are stored contiguously in a file).
- // The above implies that we currently require holding all inverted list for a query in memory.
- protected final ArrayList<IInvertedListCursor> cursorsOrderedByTokens = new ArrayList<>();
protected final InvertedListPartitions partitions = new InvertedListPartitions();
- public PartitionedTOccurrenceSearcher(IHyracksCommonContext ctx, IInPlaceInvertedIndex invIndex)
+ // To keep the current state of this search
+ protected int curPartIdx;
+ protected int endPartIdx;
+ protected int numPrefixLists;
+ protected boolean isFinalPartIdx;
+ protected boolean needToReadNewPart;
+ List<InvertedListCursor>[] partitionCursors;
+ IInvertedIndexSearchModifier searchModifier;
+
+ public PartitionedTOccurrenceSearcher(IInPlaceInvertedIndex invIndex, IHyracksTaskContext ctx)
throws HyracksDataException {
- super(ctx, invIndex);
+ super(invIndex, ctx);
initHelperTuples();
+ curPartIdx = 0;
+ endPartIdx = 0;
+ isFinalPartIdx = false;
+ isFinishedSearch = false;
+ needToReadNewPart = true;
}
private void initHelperTuples() {
@@ -87,17 +99,20 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
}
@Override
- public void search(OnDiskInvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred,
- IIndexOperationContext ictx) throws HyracksDataException {
+ public void search(IIndexCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOperationContext ictx)
+ throws HyracksDataException {
+ prepareSearch();
IPartitionedInvertedIndex partInvIndex = (IPartitionedInvertedIndex) invIndex;
- searchResult.reset();
+ finalSearchResult.reset();
if (partInvIndex.isEmpty()) {
+ isFinishedSearch = true;
resultCursor.open(null, searchPred);
return;
}
tokenizeQuery(searchPred);
short numQueryTokens = (short) queryTokenAppender.getTupleCount();
- IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier();
+
+ searchModifier = searchPred.getSearchModifier();
short numTokensLowerBound = searchModifier.getNumTokensLowerBound(numQueryTokens);
short numTokensUpperBound = searchModifier.getNumTokensUpperBound(numQueryTokens);
occurrenceThreshold = searchModifier.getOccurrenceThreshold(numQueryTokens);
@@ -108,61 +123,195 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
short maxCountPossible = numQueryTokens;
invListCursorCache.reset();
partitions.reset(numTokensLowerBound, numTokensUpperBound);
- cursorsOrderedByTokens.clear();
for (int i = 0; i < numQueryTokens; i++) {
searchKey.reset(queryTokenAppender, i);
if (!partInvIndex.openInvertedListPartitionCursors(this, ictx, numTokensLowerBound, numTokensUpperBound,
- partitions, cursorsOrderedByTokens)) {
+ partitions)) {
maxCountPossible--;
// No results possible.
if (maxCountPossible < occurrenceThreshold) {
+ // Closes all opened cursors.
+ closeCursorsInPartitions(partitions);
+ isFinishedSearch = true;
resultCursor.open(null, searchPred);
return;
}
}
}
- ArrayList<IInvertedListCursor>[] partitionCursors = partitions.getPartitions();
+
+ partitionCursors = partitions.getPartitions();
short start = partitions.getMinValidPartitionIndex();
short end = partitions.getMaxValidPartitionIndex();
- // Typically, we only enter this case for disk-based inverted indexes.
- // TODO: This behavior could potentially lead to a deadlock if we cannot pin
- // all inverted lists in memory, and are forced to wait for a page to get evicted
- // (other concurrent searchers may be in the same situation).
- // We should detect such cases, then unpin all pages, and then keep retrying to pin until we succeed.
- // This will require a different "tryPin()" mechanism in the BufferCache that will return false
- // if we'd have to wait for a page to get evicted.
- if (!cursorsOrderedByTokens.isEmpty()) {
- for (int i = start; i <= end; i++) {
+
+ // Process the partitions one-by-one.
+ endPartIdx = end;
+ for (int i = start; i <= end; i++) {
+ // Prune partition because no element in it can satisfy the occurrence threshold.
+ if (partitionCursors[i] == null) {
+ continue;
+ }
+ // Prune partition because no element in it can satisfy the occurrence threshold.
+ // An opened cursor should be closed.
+ if (partitionCursors[i].size() < occurrenceThreshold) {
+ for (InvertedListCursor cursor : partitionCursors[i]) {
+ cursor.close();
+ }
+ continue;
+ }
+ // Merge inverted lists of current partition.
+ numPrefixLists = searchModifier.getNumPrefixLists(occurrenceThreshold, partitionCursors[i].size());
+ invListMerger.reset();
+ curPartIdx = i;
+ isFinalPartIdx = i == end ? true : false;
+
+ // If the number of inverted list cursor is one, then we don't need to go through the merge process
+ // for this partition.
+ if (partitionCursors[i].size() == 1) {
+ singleInvListCursor = partitionCursors[i].get(0);
+ singleInvListCursor.prepareLoadPages();
+ singleInvListCursor.loadPages();
+ isSingleInvertedList = true;
+ needToReadNewPart = true;
+ } else {
+ singleInvListCursor = null;
+ isSingleInvertedList = false;
+ needToReadNewPart = invListMerger.merge(partitionCursors[i], occurrenceThreshold, numPrefixLists,
+ finalSearchResult);
+ searchResultBuffer = finalSearchResult.getNextFrame();
+ searchResultTupleIndex = 0;
+ searchResultFta.reset(searchResultBuffer);
+ }
+
+ // By now, some output was generated by the merger or only one cursor in this partition is associated.
+ // So, we open the cursor that will fetch these result. If it's the final partition, the outside of
+ // this for loop will handle opening of the result cursor for a single inverted list cursor case.
+ if (needToReadNewPart && isFinalPartIdx) {
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ isFinishedSearch = true;
+ }
+ resultCursor.open(null, searchPred);
+ return;
+ }
+
+ // The control reaches here if the above loop doesn't have any valid cursor.
+ isFinishedSearch = true;
+ needToReadNewPart = true;
+ resultCursor.open(null, searchPred);
+ return;
+ }
+
+ /**
+ * Continues a search process in case of the following two cases:
+ * #1. If it was paused because the output buffer of the final result was full.
+ * #2. All tuples from a single inverted list has been read.
+ *
+ * @return true only if all processing for the final list for the final partition is done.
+ * false otherwise.
+ * @throws HyracksDataException
+ */
+ @Override
+ public boolean continueSearch() throws HyracksDataException {
+ if (isFinishedSearch) {
+ return true;
+ }
+
+ // Case #1 only - output buffer was full
+ if (!needToReadNewPart) {
+ needToReadNewPart = invListMerger.continueMerge();
+ searchResultBuffer = finalSearchResult.getNextFrame();
+ searchResultTupleIndex = 0;
+ searchResultFta.reset(searchResultBuffer);
+ // Final calculation done?
+ if (needToReadNewPart && isFinalPartIdx) {
+ isFinishedSearch = true;
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ return true;
+ }
+ return false;
+ }
+
+ // Finished one partition for the both cases #1 and #2. So, moves to the next partition.
+ curPartIdx++;
+ if (curPartIdx <= endPartIdx) {
+ boolean suitablePartFound = false;
+ for (int i = curPartIdx; i <= endPartIdx; i++) {
+ // Prune partition because no element in it can satisfy the occurrence threshold.
if (partitionCursors[i] == null) {
continue;
}
// Prune partition because no element in it can satisfy the occurrence threshold.
+ // An opened cursor should be closed.
if (partitionCursors[i].size() < occurrenceThreshold) {
- cursorsOrderedByTokens.removeAll(partitionCursors[i]);
+ for (InvertedListCursor cursor : partitionCursors[i]) {
+ cursor.close();
+ }
+ continue;
}
+ suitablePartFound = true;
+ curPartIdx = i;
+ break;
+ }
+
+ // If no partition is availble to explore, we stop here.
+ if (!suitablePartFound) {
+ isFinishedSearch = true;
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ return true;
}
- // Pin all the cursors in the order of tokens.
- int numCursors = cursorsOrderedByTokens.size();
- for (int i = 0; i < numCursors; i++) {
- cursorsOrderedByTokens.get(i).pinPages();
+
+ // Merge inverted lists of current partition.
+ numPrefixLists = searchModifier.getNumPrefixLists(occurrenceThreshold, partitionCursors[curPartIdx].size());
+ invListMerger.reset();
+ finalSearchResult.resetBuffer();
+ isFinalPartIdx = curPartIdx == endPartIdx ? true : false;
+
+ // If the number of inverted list cursor is one, then we don't need to go through the merge process.
+ if (partitionCursors[curPartIdx].size() == 1) {
+ singleInvListCursor = partitionCursors[curPartIdx].get(0);
+ singleInvListCursor.prepareLoadPages();
+ singleInvListCursor.loadPages();
+ isSingleInvertedList = true;
+ needToReadNewPart = true;
+ } else {
+ singleInvListCursor = null;
+ isSingleInvertedList = false;
+ needToReadNewPart = invListMerger.merge(partitionCursors[curPartIdx], occurrenceThreshold,
+ numPrefixLists, finalSearchResult);
+ searchResultBuffer = finalSearchResult.getNextFrame();
+ searchResultTupleIndex = 0;
+ searchResultFta.reset(searchResultBuffer);
}
+
+ // Finished processing one partition
+ if (needToReadNewPart && isFinalPartIdx) {
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ isFinishedSearch = true;
+ return true;
+ }
+
+ } else {
+ isFinishedSearch = true;
}
- // Process the partitions one-by-one.
+ return false;
+ }
+
+ private void closeCursorsInPartitions(InvertedListPartitions parts) throws HyracksDataException {
+ List<InvertedListCursor>[] partCursors = parts.getPartitions();
+ short start = parts.getMinValidPartitionIndex();
+ short end = parts.getMaxValidPartitionIndex();
for (int i = start; i <= end; i++) {
- if (partitionCursors[i] == null) {
+ if (partCursors[i] == null) {
continue;
}
- // Prune partition because no element in it can satisfy the occurrence threshold.
- if (partitionCursors[i].size() < occurrenceThreshold) {
- continue;
+ for (InvertedListCursor cursor : partCursors[i]) {
+ cursor.close();
}
- // Merge inverted lists of current partition.
- int numPrefixLists = searchModifier.getNumPrefixLists(occurrenceThreshold, partitionCursors[i].size());
- invListMerger.reset();
- invListMerger.merge(partitionCursors[i], occurrenceThreshold, numPrefixLists, searchResult);
}
- resultCursor.open(null, searchPred);
}
public void setNumTokensBoundsInSearchKeys(short numTokensLowerBound, short numTokensUpperBound) {
@@ -182,7 +331,8 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
return fullHighSearchKey;
}
- public IInvertedListCursor getCachedInvertedListCursor() {
+ public InvertedListCursor getCachedInvertedListCursor() throws HyracksDataException {
return invListCursorCache.getNext();
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java
deleted file mode 100644
index 2da1434..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.invertedindex.search;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.hyracks.api.context.IHyracksCommonContext;
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAccessor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeFrameTupleAppender;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeTupleReference;
-
-/**
- * Byte-buffer backed storage for intermediate and final results of inverted-index searches.
- */
-// TODO: Rename members.
-public class SearchResult {
- protected final ArrayList<ByteBuffer> buffers = new ArrayList<ByteBuffer>();
- protected final IHyracksCommonContext ctx;
- protected final FixedSizeFrameTupleAppender appender;
- protected final FixedSizeFrameTupleAccessor accessor;
- protected final FixedSizeTupleReference tuple;
- protected final ITypeTraits[] typeTraits;
- protected final int invListElementSize;
-
- protected int currBufIdx;
- protected int numResults;
-
- public SearchResult(ITypeTraits[] invListFields, IHyracksCommonContext ctx) throws HyracksDataException {
- typeTraits = new ITypeTraits[invListFields.length + 1];
- int tmp = 0;
- for (int i = 0; i < invListFields.length; i++) {
- typeTraits[i] = invListFields[i];
- tmp += invListFields[i].getFixedLength();
- }
- invListElementSize = tmp;
- // Integer for counting occurrences.
- typeTraits[invListFields.length] = IntegerPointable.TYPE_TRAITS;
- this.ctx = ctx;
- appender = new FixedSizeFrameTupleAppender(ctx.getInitialFrameSize(), typeTraits);
- accessor = new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), typeTraits);
- tuple = new FixedSizeTupleReference(typeTraits);
- buffers.add(ctx.allocateFrame());
- }
-
- /**
- * Initialize from other search-result object to share member instances except for result buffers.
- *
- * @throws HyracksDataException
- */
- public SearchResult(SearchResult other) throws HyracksDataException {
- this.ctx = other.ctx;
- this.appender = other.appender;
- this.accessor = other.accessor;
- this.tuple = other.tuple;
- this.typeTraits = other.typeTraits;
- this.invListElementSize = other.invListElementSize;
- buffers.add(ctx.allocateFrame());
- }
-
- public FixedSizeFrameTupleAccessor getAccessor() {
- return accessor;
- }
-
- public FixedSizeFrameTupleAppender getAppender() {
- return appender;
- }
-
- public FixedSizeTupleReference getTuple() {
- return tuple;
- }
-
- public ArrayList<ByteBuffer> getBuffers() {
- return buffers;
- }
-
- public void reset() {
- currBufIdx = 0;
- numResults = 0;
- appender.reset(buffers.get(0), true);
- }
-
- public void clear() {
- currBufIdx = 0;
- numResults = 0;
- for (ByteBuffer buffer : buffers) {
- appender.reset(buffer, true);
- }
- }
-
- public void append(ITupleReference invListElement, int count) throws HyracksDataException {
- ByteBuffer currentBuffer = buffers.get(currBufIdx);
- if (!appender.hasSpace()) {
- currBufIdx++;
- if (currBufIdx >= buffers.size()) {
- buffers.add(ctx.allocateFrame());
- }
- currentBuffer = buffers.get(currBufIdx);
- appender.reset(currentBuffer, true);
- }
- // Append inverted-list element.
- if (!appender.append(invListElement.getFieldData(0), invListElement.getFieldStart(0), invListElementSize)) {
- throw new IllegalStateException();
- }
- // Append count.
- if (!appender.append(count)) {
- throw new IllegalStateException();
- }
- appender.incrementTupleCount(1);
- numResults++;
- }
-
- public int getCurrentBufferIndex() {
- return currBufIdx;
- }
-
- public ITypeTraits[] getTypeTraits() {
- return typeTraits;
- }
-
- public int getNumResults() {
- return numResults;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
index 4c9f037..9808ae1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java
@@ -21,26 +21,30 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.search;
import java.util.ArrayList;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexSearchCursor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
+import org.apache.hyracks.storage.common.IIndexCursor;
+/**
+ * Conducts T-Occurrence searches on inverted lists.
+ */
public class TOccurrenceSearcher extends AbstractTOccurrenceSearcher {
- protected final ArrayList<IInvertedListCursor> invListCursors = new ArrayList<>();
+ protected final ArrayList<InvertedListCursor> invListCursors = new ArrayList<>();
- public TOccurrenceSearcher(IHyracksCommonContext ctx, IInPlaceInvertedIndex invIndex) throws HyracksDataException {
- super(ctx, invIndex);
+ public TOccurrenceSearcher(IInPlaceInvertedIndex invIndex, IHyracksTaskContext ctx) throws HyracksDataException {
+ super(invIndex, ctx);
}
@Override
- public void search(OnDiskInvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred,
- IIndexOperationContext ictx) throws HyracksDataException {
+ public void search(IIndexCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOperationContext ictx)
+ throws HyracksDataException {
+ prepareSearch();
tokenizeQuery(searchPred);
int numQueryTokens = queryTokenAppender.getTupleCount();
@@ -48,7 +52,7 @@ public class TOccurrenceSearcher extends AbstractTOccurrenceSearcher {
invListCursorCache.reset();
for (int i = 0; i < numQueryTokens; i++) {
searchKey.reset(queryTokenAppender, i);
- IInvertedListCursor invListCursor = invListCursorCache.getNext();
+ InvertedListCursor invListCursor = invListCursorCache.getNext();
invIndex.openInvertedListCursor(invListCursor, searchKey, ictx);
invListCursors.add(invListCursor);
}
@@ -60,8 +64,54 @@ public class TOccurrenceSearcher extends AbstractTOccurrenceSearcher {
}
int numPrefixLists = searchModifier.getNumPrefixLists(occurrenceThreshold, invListCursors.size());
- searchResult.reset();
- invListMerger.merge(invListCursors, occurrenceThreshold, numPrefixLists, searchResult);
+ // For a single inverted list case, we don't need to call merge() method since elements from a single inverted
+ // list cursor will be the final answer.
+ if (numQueryTokens == 1 && occurrenceThreshold == 1) {
+ singleInvListCursor = invListCursors.get(0);
+ singleInvListCursor.prepareLoadPages();
+ singleInvListCursor.loadPages();
+ isSingleInvertedList = true;
+ isFinishedSearch = true;
+ } else {
+ finalSearchResult.reset();
+ isFinishedSearch =
+ invListMerger.merge(invListCursors, occurrenceThreshold, numPrefixLists, finalSearchResult);
+ searchResultBuffer = finalSearchResult.getNextFrame();
+ searchResultTupleIndex = 0;
+ searchResultFta.reset(searchResultBuffer);
+ }
+
+ if (isFinishedSearch) {
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ }
+ // Some or all output was generated by the merger. Let the result cursor fetch the output.
resultCursor.open(null, searchPred);
}
+
+ /**
+ * Continues a search process if it was paused because the output buffer (one frame) of the final result was full.
+ * This method should not be called for a single inverted list case since there cannot be multiple inverted list
+ * cursors for a single keyword.
+ *
+ * @return true only if all processing for the final list is done.
+ * false otherwise.
+ * @throws HyracksDataException
+ */
+ @Override
+ public boolean continueSearch() throws HyracksDataException {
+ if (isFinishedSearch) {
+ return true;
+ }
+ isFinishedSearch = invListMerger.continueMerge();
+ searchResultBuffer = finalSearchResult.getNextFrame();
+ searchResultTupleIndex = 0;
+ searchResultFta.reset(searchResultBuffer);
+ if (isFinishedSearch) {
+ invListMerger.close();
+ finalSearchResult.finalizeWrite();
+ }
+ return isFinishedSearch;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/ObjectCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/ObjectCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/ObjectCache.java
index 01c74c9..ccfe1b8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/ObjectCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/ObjectCache.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.util;
import java.util.ArrayList;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IObjectFactory;
public class ObjectCache<T> {
@@ -29,14 +30,14 @@ public class ObjectCache<T> {
protected final ArrayList<T> cache;
protected int lastReturned = 0;
- public ObjectCache(IObjectFactory<T> objFactory, int initialSize, int expandSize) {
+ public ObjectCache(IObjectFactory<T> objFactory, int initialSize, int expandSize) throws HyracksDataException {
this.objFactory = objFactory;
this.cache = new ArrayList<T>(initialSize);
this.expandSize = expandSize;
expand(initialSize);
}
- private void expand(int expandSize) {
+ private void expand(int expandSize) throws HyracksDataException {
for (int i = 0; i < expandSize; i++) {
cache.add(objFactory.create());
}
@@ -46,7 +47,7 @@ public class ObjectCache<T> {
lastReturned = 0;
}
- public T getNext() {
+ public T getNext() throws HyracksDataException {
if (lastReturned >= cache.size()) {
expand(expandSize);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index ee37043..0dc25ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -51,9 +51,8 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.rtree.frames.RTreeFrameFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.trace.ITracer;
@@ -229,11 +228,10 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
}
@Override
- protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback,
- ISearchOperationCallback searchCallback) {
+ protected LSMRTreeOpContext createOpContext(IIndexAccessParameters iap) {
return new LSMRTreeOpContext(this, memoryComponents, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
- btreeLeafFrameFactory, modCallback, searchCallback, getTreeFields(), getFilterFields(), getHarness(),
- comparatorFields, linearizerArray, getFilterCmpFactories(), tracer);
+ btreeLeafFrameFactory, iap.getModificationCallback(), iap.getSearchOperationCallback(), getTreeFields(),
+ getFilterFields(), getHarness(), comparatorFields, linearizerArray, getFilterCmpFactories(), tracer);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index d891d9e..7c1467c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -374,8 +374,7 @@ public class LSMRTree extends AbstractLSMRTree {
@Override
public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) {
- return new LSMRTreeAccessor(getHarness(),
- createOpContext(iap.getModificationCallback(), iap.getSearchOperationCallback()), buddyBTreeFields);
+ return new LSMRTreeAccessor(getHarness(), createOpContext(iap), buddyBTreeFields);
}
// This function is modified for R-Trees without antimatter tuples to allow buddy B-Tree to have only primary keys
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index cade80f..3bb94ed 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -260,7 +260,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
@Override
public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) {
- LSMRTreeOpContext opCtx = createOpContext(iap.getModificationCallback(), iap.getSearchOperationCallback());
+ LSMRTreeOpContext opCtx = createOpContext(iap);
return new LSMTreeIndexAccessor(getHarness(), opCtx, cursorFactory);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index 886285c..18ced6d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -30,6 +30,7 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePush
import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.am.rtree.util.RTreeUtils;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -88,4 +89,10 @@ public class RTreeSearchOperatorNodePushable extends IndexSearchOperatorNodePush
protected int getFieldCount() {
return ((ITreeIndex) index).getFieldCount();
}
+
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ // no additional parameteres are required for the B+Tree search case yet
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessor.java
index 382aea2..0cb1b29 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndexAccessor.java
@@ -83,8 +83,10 @@ public interface IIndexAccessor extends IDestroyable {
/**
* Creates a cursor appropriate for passing into search().
*
+ * @throws HyracksDataException
+ *
*/
- IIndexCursor createSearchCursor(boolean exclusive);
+ IIndexCursor createSearchCursor(boolean exclusive) throws HyracksDataException;
/**
* Open the given cursor for an index search using the given predicate as
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexTestWorker.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexTestWorker.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexTestWorker.java
index b9123a5..830bab5 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexTestWorker.java
@@ -37,7 +37,7 @@ public abstract class AbstractIndexTestWorker extends Thread implements ITreeInd
private final TestOperationSelector opSelector;
private final int numBatches;
- protected final IIndexAccessor indexAccessor;
+ protected IIndexAccessor indexAccessor;
public AbstractIndexTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, IIndex index,
int numBatches) throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java
index 2c08ba0..effeafc 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/IndexTestContext.java
@@ -35,7 +35,7 @@ public abstract class IndexTestContext<T extends CheckTuple> implements IIndexTe
protected final IIndex index;
protected final ArrayTupleBuilder tupleBuilder;
protected final ArrayTupleReference tuple = new ArrayTupleReference();
- protected final IIndexAccessor indexAccessor;
+ protected IIndexAccessor indexAccessor;
public IndexTestContext(ISerializerDeserializer[] fieldSerdes, IIndex index, boolean filtered)
throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/config/AccessMethodTestsConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/config/AccessMethodTestsConfig.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/config/AccessMethodTestsConfig.java
index 60e10ad..1e462f8 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/config/AccessMethodTestsConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/config/AccessMethodTestsConfig.java
@@ -85,6 +85,9 @@ public class AccessMethodTestsConfig {
public static final int LSM_INVINDEX_HYRACKS_FRAME_SIZE = 32768;
public static final double LSM_INVINDEX_BLOOMFILTER_FALSE_POSITIVE_RATE = 0.01;
public static final int LSM_INVINDEX_NUM_MUTABLE_COMPONENTS = 2;
+ // inverted-index-search frame limit for the test purposes (minimum 5)
+ // (1 for query token frame, 2 for intermediate results, 1 for the final result, 1 for inverted lists)
+ public static final int LSM_INVINDEX_SEARCH_FRAME_LIMIT = 5;
// Test parameters.
public static final int LSM_INVINDEX_NUM_DOCS_TO_INSERT = 100;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreePointSearchCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreePointSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreePointSearchCursorTest.java
index 18d89ca..b16daf5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreePointSearchCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreePointSearchCursorTest.java
@@ -113,7 +113,7 @@ public class LSMBTreePointSearchCursorTest extends IIndexCursorTest {
@Override
protected IIndexCursor createCursor(IIndexAccessor accessor) {
- opCtx = lsmBtree.createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ opCtx = lsmBtree.createOpContext(NoOpIndexAccessParameters.INSTANCE);
return new LSMBTreePointSearchCursor(opCtx);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeRangeSearchCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeRangeSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeRangeSearchCursorTest.java
index e436596..e6850aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeRangeSearchCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/cursor/LSMBTreeRangeSearchCursorTest.java
@@ -94,7 +94,7 @@ public class LSMBTreeRangeSearchCursorTest extends IIndexCursorTest {
@Override
protected IIndexCursor createCursor(IIndexAccessor accessor) {
- opCtx = lsmBtree.createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ opCtx = lsmBtree.createOpContext(NoOpIndexAccessParameters.INSTANCE);
return new LSMBTreeRangeSearchCursor(opCtx);
}