You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wa...@apache.org on 2018/02/19 17:07:52 UTC
[5/7] asterixdb git commit: [ASTERIXDB-2083][COMP][RT][IDX][SITE]
Budget-Constrained Inverted index search
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index c15704a..ad715a4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -118,17 +118,20 @@ public class APIFramework {
private static final int MIN_FRAME_LIMIT_FOR_SORT = 3;
private static final int MIN_FRAME_LIMIT_FOR_GROUP_BY = 4;
private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5;
+ // one for query, two for intermediate results, one for final result, and one for reading an inverted list
+ private static final int MIN_FRAME_LIMIT_FOR_TEXTSEARCH = 5;
private static final String LPLAN = "Logical plan";
private static final String OPLAN = "Optimized logical plan";
// A white list of supported configurable parameters.
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
- CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
- FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME,
- FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION,
- FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, "inline_with",
- "hash_merge", "output-record-type", AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION);
+ CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
+ CompilerProperties.COMPILER_PARALLELISM_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
+ StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
+ FeedActivityDetails.COLLECT_LOCATIONS, "inline_with", "hash_merge", "output-record-type",
+ AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
@@ -341,11 +344,16 @@ public class APIFramework {
int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY,
querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY),
compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN);
+ int textSearchFrameLimit = getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
+ querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY),
+ compilerProperties.getTextSearchMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_TEXTSEARCH);
final PhysicalOptimizationConfig physOptConf = OptimizationConfUtil.getPhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
physOptConf.setMaxFramesExternalGroupBy(groupFrameLimit);
physOptConf.setMaxFramesForJoin(joinFrameLimit);
+ physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
+
return physOptConf;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
index 7eaaa3d..1913683 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
@@ -22,6 +22,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
public class OperatorResourcesComputer {
@@ -33,14 +34,16 @@ public class OperatorResourcesComputer {
private final long groupByMemorySize;
private final long joinMemorySize;
private final long sortMemorySize;
+ private final long textSearchMemorySize;
private final long frameSize;
public OperatorResourcesComputer(int numComputationPartitions, int sortFrameLimit, int groupFrameLimit,
- int joinFrameLimit, long frameSize) {
+ int joinFrameLimit, int textSearchFrameLimit, long frameSize) {
this.numComputationPartitions = numComputationPartitions;
this.groupByMemorySize = groupFrameLimit * frameSize;
this.joinMemorySize = joinFrameLimit * frameSize;
this.sortMemorySize = sortFrameLimit * frameSize;
+ this.textSearchMemorySize = textSearchFrameLimit * frameSize;
this.frameSize = frameSize;
}
@@ -62,7 +65,6 @@ public class OperatorResourcesComputer {
case EMPTYTUPLESOURCE:
case DELEGATE_OPERATOR:
case EXTERNAL_LOOKUP:
- case LEFT_OUTER_UNNEST_MAP:
case LIMIT:
case MATERIALIZE:
case NESTEDTUPLESOURCE:
@@ -78,7 +80,6 @@ public class OperatorResourcesComputer {
case UNIONALL:
case UNNEST:
case LEFT_OUTER_UNNEST:
- case UNNEST_MAP:
case UPDATE:
case WRITE:
case WRITE_RESULT:
@@ -86,6 +87,15 @@ public class OperatorResourcesComputer {
case INSERT_DELETE_UPSERT:
case INTERSECT:
return getOperatorRequiredMemory(operator, frameSize);
+ case LEFT_OUTER_UNNEST_MAP:
+ case UNNEST_MAP:
+ // Since an inverted-index search requires certain amount of memory, needs to calculate
+ // the memory size differently if the given index-search is an inverted-index search.
+ long unnestMapMemorySize = frameSize;
+ if (isInvertedIndexSearch((AbstractUnnestMapOperator) operator)) {
+ unnestMapMemorySize += textSearchMemorySize;
+ }
+ return getOperatorRequiredMemory(operator, unnestMapMemorySize);
case EXCHANGE:
return getExchangeRequiredMemory((ExchangeOperator) operator);
case GROUP:
@@ -108,6 +118,16 @@ public class OperatorResourcesComputer {
return memorySize;
}
+ private boolean isInvertedIndexSearch(AbstractUnnestMapOperator op) {
+ IPhysicalOperator physicalOperator = op.getPhysicalOperator();
+ final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag();
+ if (physicalOperatorTag == PhysicalOperatorTag.LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH
+ || physicalOperatorTag == PhysicalOperatorTag.SINGLE_PARTITION_INVERTED_INDEX_SEARCH) {
+ return true;
+ }
+ return false;
+ }
+
private long getExchangeRequiredMemory(ExchangeOperator op) {
final IPhysicalOperator physicalOperator = op.getPhysicalOperator();
final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
index 0149ffa..d9ead33 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java
@@ -58,9 +58,10 @@ public class ResourceUtils {
final int sortFrameLimit = physicalOptimizationConfig.getMaxFramesExternalSort();
final int groupFrameLimit = physicalOptimizationConfig.getMaxFramesForGroupBy();
final int joinFrameLimit = physicalOptimizationConfig.getMaxFramesForJoin();
+ final int textSearchFrameLimit = physicalOptimizationConfig.getMaxFramesForTextSearch();
final List<PlanStage> planStages = getStages(plan);
return getStageBasedRequiredCapacity(planStages, computationLocations.getLocations().length, sortFrameLimit,
- groupFrameLimit, joinFrameLimit, frameSize);
+ groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize);
}
public static List<PlanStage> getStages(ILogicalPlan plan) throws AlgebricksException {
@@ -72,9 +73,9 @@ public class ResourceUtils {
}
public static IClusterCapacity getStageBasedRequiredCapacity(List<PlanStage> stages, int computationLocations,
- int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int frameSize) {
+ int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int textSearchFrameLimit, int frameSize) {
final OperatorResourcesComputer computer = new OperatorResourcesComputer(computationLocations, sortFrameLimit,
- groupFrameLimit, joinFrameLimit, frameSize);
+ groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize);
final IClusterCapacity clusterCapacity = new ClusterCapacity();
final Long maxRequiredMemory = stages.stream().mapToLong(stage -> stage.getRequiredMemory(computer)).max()
.orElseThrow(IllegalStateException::new);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index fe6cf64..0d9f54f 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc2.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 113497d..ddf1438 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
compiler.parallelism=-1
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/main/resources/cc3.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index e210bf62..b819f24 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -51,6 +51,7 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
compiler.parallelism=3
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
index 0e55b1e..d3113ca 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/resource/PlanStagesGeneratorTest.java
@@ -301,7 +301,7 @@ public class PlanStagesGeneratorTest {
private void assertRequiredMemory(List<PlanStage> stages, long expectedMemory) {
final IClusterCapacity clusterCapacity = ResourceUtils.getStageBasedRequiredCapacity(stages, PARALLELISM,
- FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_SIZE);
+ FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_SIZE);
Assert.assertEquals(clusterCapacity.getAggregatedMemoryByteSize(), expectedMemory);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 5cf1bbe..1c141f1 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -52,5 +52,6 @@ compiler.framesize=32KB
compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
new file mode 100644
index 0000000..22ad574
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.1.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Full-text search index test
+ * : This test is intended to verify that the full-text search works as expected.
+ * : query #3 - two string values in [an ordered list] query with "any" option
+ * : in this case, "any" option that enforces a disjunctive search will be applied.
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index fulltext_index_title on MyData(title) type fulltext;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
new file mode 100644
index 0000000..9b7c1f6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.2.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset MyData
+using localfs
+(("path"="asterix_nc1://data/spatial/spatialData2.json"),("format"="adm"));
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
new file mode 100644
index 0000000..28c4190
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-03/fulltext-index-03.3.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select element {"id":ftval.id}
+from MyData as ftval
+where test.ftcontains(ftval.title, ["object","database"], {"mode":"any"})
+order by ftval.id;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
new file mode 100644
index 0000000..e63ecdb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.1.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Testing using a secondary fulltext index
+ * : Based on the text.searchmemory budget (default=160KB), this test should
+ * : create intermediate result files.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+// For bulk-loading test (tuples will be fetched from the disk)
+create dataset Fragile_raw (FragileTypeRaw) primary key row_id;
+
+// For insert test (tuples will be fetched from memory)
+create dataset Fragile_raw_insert (FragileTypeRaw) primary key row_id;
+create index fulltext_insert_index on Fragile_raw_insert(date) type fulltext;
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
new file mode 100644
index 0000000..3333df8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Fragile_raw using localfs
+((`path`=`asterix_nc1://data/csv/fragile_sample.csv`),(`format`=`delimited-text`),(`delimiter`=`,`)) pre-sorted;
+
+insert into Fragile_raw_insert (
+ select value t from Fragile_raw t
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
new file mode 100644
index 0000000..5d719c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on Fragile_raw(date) type ngram(2);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
new file mode 100644
index 0000000..a41a95f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value count(*)
+from Fragile_raw as o
+where ftcontains(o.date,'20130417');
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
new file mode 100644
index 0000000..81dd172
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value count(*)
+from Fragile_raw_insert as o
+where ftcontains(o.date,'20130417');
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
new file mode 100644
index 0000000..b7f0ec5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.1.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Testing using a secondary inverted n-gram index
+ * : Based on the text.searchmemory budget (default=160KB), this test should
+ * : create intermediate result files.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type FragileTypeRaw as closed {
+ row_id: int32,
+ sid: int32,
+ date: string,
+ day: int32,
+ time: string,
+ bpm: int32,
+ RR: float
+};
+
+// For bulk-loading test (tuples will be fetched from the disk)
+create dataset Fragile_raw (FragileTypeRaw) primary key row_id;
+
+// For insert test (tuples will be fetched from memory)
+create dataset Fragile_raw_insert (FragileTypeRaw) primary key row_id;
+create index ngram_insert_index on Fragile_raw_insert(date) type ngram(2);
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
new file mode 100644
index 0000000..3333df8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset Fragile_raw using localfs
+((`path`=`asterix_nc1://data/csv/fragile_sample.csv`),(`format`=`delimited-text`),(`delimiter`=`,`)) pre-sorted;
+
+insert into Fragile_raw_insert (
+ select value t from Fragile_raw t
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
new file mode 100644
index 0000000..5d719c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+create index ngram_index on Fragile_raw(date) type ngram(2);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
new file mode 100644
index 0000000..64a71ce
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The following predicate returns all rows since there are only two distinct date values on the dataset.
+
+use test;
+
+select value count(*)
+from Fragile_raw as o
+where `edit-distance-check`(o.date,'20130417',4)[0];
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
new file mode 100644
index 0000000..c68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The following predicate returns all rows since there are only two distinct date values on the dataset.
+
+use test;
+
+select value count(*)
+from Fragile_raw_insert as o
+where `edit-distance-check`(o.date,'20130417',4)[0];
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 59057b1..28ef0eb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : 0,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "INFO",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 13dacb8..61cf528 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : -1,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "WARN",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9f18638..d340386 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -11,6 +11,7 @@
"compiler\.joinmemory" : 262144,
"compiler\.parallelism" : 3,
"compiler\.sortmemory" : 327680,
+ "compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
"instance\.name" : "DEFAULT_INSTANCE",
"log\.level" : "WARN",
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
new file mode 100644
index 0000000..1746da6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.4.adm
@@ -0,0 +1 @@
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
new file mode 100644
index 0000000..1746da6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/fulltext/fulltext-index-large-data/fulltext-index-large-data.5.adm
@@ -0,0 +1 @@
+10000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.4.adm
@@ -0,0 +1 @@
+20000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/index-selection/inverted-index-ngram-edit-distance-large-data/inverted-index-ngram-edit-distance-large-data.5.adm
@@ -0,0 +1 @@
+20000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 961fb41..909b523 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -2744,10 +2744,20 @@
</compilation-unit>
</test-case>
<test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-03">
+ <output-dir compare="Text">fulltext-index-03</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="fulltext">
<compilation-unit name="fulltext-index-08">
<output-dir compare="Text">fulltext-index-08</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="fulltext">
+ <compilation-unit name="fulltext-index-large-data">
+ <output-dir compare="Text">fulltext-index-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="global-aggregate">
<test-case FilePath="global-aggregate">
@@ -3169,6 +3179,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="index-selection">
+ <compilation-unit name="inverted-index-ngram-edit-distance-large-data">
+ <output-dir compare="Text">inverted-index-ngram-edit-distance-large-data</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="index-selection">
<compilation-unit name="inverted-index-ngram-edit-distance-word-tokens">
<output-dir compare="Text">inverted-index-ngram-edit-distance-word-tokens</output-dir>
</compilation-unit>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 0580c1f..6d89ff5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -42,6 +42,10 @@ public class CompilerProperties extends AbstractProperties {
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
"The memory budget (in bytes) for a group by operator instance in a partition"),
+ COMPILER_TEXTSEARCHMEMORY(
+ LONG_BYTE_UNIT,
+ StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
+ "The memory budget (in bytes) for an inverted-index-search operator instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@ -98,6 +102,8 @@ public class CompilerProperties extends AbstractProperties {
public static final String COMPILER_JOINMEMORY_KEY = Option.COMPILER_JOINMEMORY.ini();
+ public static final String COMPILER_TEXTSEARCHMEMORY_KEY = Option.COMPILER_TEXTSEARCHMEMORY.ini();
+
public static final String COMPILER_PARALLELISM_KEY = Option.COMPILER_PARALLELISM.ini();
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
@@ -118,6 +124,10 @@ public class CompilerProperties extends AbstractProperties {
return accessor.getLong(Option.COMPILER_GROUPMEMORY);
}
+ public long getTextSearchMemorySize() {
+ return accessor.getLong(Option.COMPILER_TEXTSEARCHMEMORY);
+ }
+
public int getFrameSize() {
return accessor.getInt(Option.COMPILER_FRAMESIZE);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/asterixdb/asterix-doc/src/site/markdown/ncservice.md
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-doc/src/site/markdown/ncservice.md b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
index aadecc2..1d174e9 100644
--- a/asterixdb/asterix-doc/src/site/markdown/ncservice.md
+++ b/asterixdb/asterix-doc/src/site/markdown/ncservice.md
@@ -345,6 +345,7 @@ The following parameters are configured under the "[common]" section.
| common | compiler.joinmemory | The memory budget (in bytes) for a join operator instance in a partition | 33554432 (32 MB) |
| common | compiler.parallelism | The degree of parallelism for query execution. Zero means to use the storage parallelism as the query execution parallelism, while other integer values dictate the number of query execution parallel partitions. The system will fall back to use the number of all available CPU cores in the cluster as the degree of parallelism if the number set by a user is too large or too small | 0 |
| common | compiler.sortmemory | The memory budget (in bytes) for a sort operator instance in a partition | 33554432 (32 MB) |
+| common | compiler.textsearchmemory | The memory budget (in bytes) for an inverted-index-search operator instance in a partition | 33554432 (32 MB) |
| common | instance.name | The name of this cluster instance | DEFAULT_INSTANCE |
| common | log.level | The logging level for master and slave processes | WARNING |
| common | max.wait.active.cluster | The max pending time (in seconds) for cluster startup. After the threshold, if the cluster still is not up and running, it is considered unavailable | 60 |
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index a242927..07ff0ab 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -28,6 +28,7 @@ public class PhysicalOptimizationConfig {
private static final String MAX_FRAMES_EXTERNAL_GROUP_BY = "MAX_FRAMES_EXTERNAL_GROUP_BY";
private static final String MAX_FRAMES_FOR_JOIN_LEFT_INPUT = "MAX_FRAMES_FOR_JOIN_LEFT_INPUT";
private static final String MAX_FRAMES_FOR_JOIN = "MAX_FRAMES_FOR_JOIN";
+ private static final String MAX_FRAMES_FOR_TEXTSEARCH = "MAX_FRAMES_FOR_TEXTSEARCH";
private static final String FUDGE_FACTOR = "FUDGE_FACTOR";
private static final String MAX_RECORDS_PER_FRAME = "MAX_RECORDS_PER_FRAME";
@@ -109,6 +110,15 @@ public class PhysicalOptimizationConfig {
setInt(MAX_FRAMES_EXTERNAL_SORT, frameLimit);
}
+ public int getMaxFramesForTextSearch() {
+ int frameSize = getFrameSize();
+ return getInt(MAX_FRAMES_FOR_TEXTSEARCH, (int) (((long) 32 * MB) / frameSize));
+ }
+
+ public void setMaxFramesForTextSearch(int frameLimit) {
+ setInt(MAX_FRAMES_FOR_TEXTSEARCH, frameLimit);
+ }
+
public int getHashGroupByTableSize() {
return getInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index ad2d6f9..0dca782 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -142,6 +142,11 @@ public class ErrorCode {
public static final int CANNOT_DEACTIVATE_PINNED_BLOOM_FILTER = 106;
public static final int PREDICATE_CANNOT_BE_NULL = 107;
public static final int FULLTEXT_ONLY_EXECUTABLE_FOR_STRING_OR_LIST = 108;
+ public static final int NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH = 109;
+ public static final int CANNOT_CONTINUE_TEXT_SEARCH_HYRACKS_TASK_IS_NULL = 110;
+ public static final int CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL = 111;
+ public static final int CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT = 112;
+ public static final int UNDEFINED_INVERTED_LIST_MERGE_TYPE = 113;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
index 5609721..9314f24 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksConstants.java
@@ -21,6 +21,10 @@ package org.apache.hyracks.api.util;
public class HyracksConstants {
public static final String KEY_MESSAGE = "HYX:MSG";
public static final String HYRACKS_LOGGER_NAME = "org.apache.hyracks";
+ // A frame manager that manages all inverted index searches
+ public static final String INVERTED_INDEX_SEARCH_FRAME_MANAGER = "INVERTED_INDEX_SEARCH_FRAME_MANAGER";
+ // Hyracks task context
+ public static final String HYRACKS_TASK_CONTEXT = "HYRACKS_TASK_CONTEXT";
private HyracksConstants() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index c3945a3..50e1ad4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -125,5 +125,10 @@
106 = Failed to deactivate the bloom filter since it is pinned by other users
107 = The given search predicate can't be null.
108 = Full-text search can be only executed on STRING or (UN)ORDERED LIST.
+109 = Not enough memory is allocated to %1$s. Please assign more memory to text search.
+110 = To conduct an inverted-index search, the given Hyracks task context cannot be null.
+111 = To conduct an inverted-index search, the given buffer manager cannot be null.
+112 = Cannot add an element to an inverted-index search result.
+113 = Undefined inverted-list merge type: %1$s
10000 = The given rule collection %1$s is not an instance of the List class.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
new file mode 100644
index 0000000..bf5e803
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/BufferManagerBackedVSizeFrame.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.context.IHyracksFrameMgrContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Variable size frame managed by a buffer manager. The frame size can be changed.
+ * The caller of this class needs to ensure that this class releases its buffer by calling close() method.
+ */
+public class BufferManagerBackedVSizeFrame implements IFrame {
+
+ protected final int minFrameSize;
+ protected final IHyracksFrameMgrContext ctx;
+ protected final ISimpleFrameBufferManager bufferManager;
+ protected ByteBuffer frame;
+
+ public BufferManagerBackedVSizeFrame(IHyracksFrameMgrContext ctx, ISimpleFrameBufferManager bufferManager)
+ throws HyracksDataException {
+ this(ctx, ctx.getInitialFrameSize(), bufferManager);
+ }
+
+ public BufferManagerBackedVSizeFrame(IHyracksFrameMgrContext ctx, int frameSize,
+ ISimpleFrameBufferManager bufferManager) throws HyracksDataException {
+ this.ctx = ctx;
+ this.minFrameSize = ctx.getInitialFrameSize();
+ this.bufferManager = bufferManager;
+ frame = bufferManager.acquireFrame(frameSize);
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return frame;
+ }
+
+ @Override
+ public void ensureFrameSize(int newSize) throws HyracksDataException {
+ if (newSize > getFrameSize()) {
+ bufferManager.releaseFrame(frame);
+ frame = bufferManager.acquireFrame(newSize);
+ }
+ }
+
+ @Override
+ public void resize(int frameSize) throws HyracksDataException {
+ if (getFrameSize() != frameSize) {
+ bufferManager.releaseFrame(frame);
+ frame = bufferManager.acquireFrame(frameSize);
+ }
+ }
+
+ @Override
+ public int getFrameSize() {
+ return frame.capacity();
+ }
+
+ @Override
+ public int getMinSize() {
+ return minFrameSize;
+ }
+
+ @Override
+ public void reset() throws HyracksDataException {
+ resize(minFrameSize);
+ Arrays.fill(frame.array(), (byte) 0);
+ frame.clear();
+ }
+
+ /**
+ * Allocates an initial frame and clears it.
+ */
+ public void acquireFrame() throws HyracksDataException {
+ if (frame == null) {
+ frame = bufferManager.acquireFrame(minFrameSize);
+ }
+ reset();
+ }
+
+ public void destroy() throws HyracksDataException {
+ if (frame != null) {
+ bufferManager.releaseFrame(frame);
+ frame = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 72941f3..6bbf437 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -104,4 +105,10 @@ public class BTreeSearchOperatorNodePushable extends IndexSearchOperatorNodePush
protected int getFieldCount() {
return ((ITreeIndex) index).getFieldCount();
}
+
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ // No additional parameters are required for the B+Tree search case
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index ab9359f..eb493af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -144,7 +144,10 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
protected abstract void resetSearchPredicate(int tupleIndex);
- protected IIndexCursor createCursor() {
+ // Assigns any index-type specific related accessor parameters
+ protected abstract void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException;
+
+ protected IIndexCursor createCursor() throws HyracksDataException {
return indexAccessor.createSearchCursor(false);
}
@@ -184,6 +187,7 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
ISearchOperationCallback searchCallback =
searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
IIndexAccessParameters iap = new IndexAccessParameters(NoOpOperationCallback.INSTANCE, searchCallback);
+ addAdditionalIndexAccessorParams(iap);
indexAccessor = index.createAccessor(iap);
cursor = createCursor();
if (retainInput) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
index 3618cae..2edea70 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpIndexAccessParameters.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.common.impls;
+import java.util.Collections;
import java.util.Map;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
@@ -26,6 +27,8 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
public class NoOpIndexAccessParameters implements IIndexAccessParameters {
public static final NoOpIndexAccessParameters INSTANCE = new NoOpIndexAccessParameters();
+ // Immutable empty map not to cause getParameters().get() generate an exception
+ private static final Map<String, Object> paramMap = Collections.emptyMap();
private NoOpIndexAccessParameters() {
}
@@ -42,7 +45,7 @@ public class NoOpIndexAccessParameters implements IIndexAccessParameters {
@Override
public Map<String, Object> getParameters() {
- return null;
+ return paramMap;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
index 6a3fdf1..418b370 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/test/java/org/apache/hyracks/storage/am/common/test/IIndexCursorTest.java
@@ -279,7 +279,7 @@ public abstract class IIndexCursorTest {
Assert.assertNull(cursor.getTuple());
}
- protected IIndexCursor createCursor(IIndexAccessor accessor) {
+ protected IIndexCursor createCursor(IIndexAccessor accessor) throws Exception {
return accessor.createSearchCursor(false);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
index 1f71876..860d36e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDiskComponentScanOperatorNodePushable.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
public class LSMBTreeDiskComponentScanOperatorNodePushable extends IndexSearchOperatorNodePushable {
@@ -66,4 +67,9 @@ public class LSMBTreeDiskComponentScanOperatorNodePushable extends IndexSearchOp
return ((ITreeIndex) index).getFieldCount() + 2;
}
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) throws HyracksDataException {
+ // no additional parameters are required for the B+Tree search case
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 0be5556..6e06d37 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -61,7 +61,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -633,8 +632,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- protected AbstractLSMIndexOperationContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ protected AbstractLSMIndexOperationContext createOpContext(IIndexAccessParameters iap) {
return null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 4610fd2..4578eb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -65,8 +65,6 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICurs
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -394,18 +392,17 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
@Override
- public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ public LSMBTreeOpContext createOpContext(IIndexAccessParameters iap) {
int numBloomFilterKeyFields = hasBloomFilter
? ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length : 0;
return new LSMBTreeOpContext(this, memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
- modificationCallback, searchCallback, numBloomFilterKeyFields, getTreeFields(), getFilterFields(),
- getHarness(), getFilterCmpFactories(), tracer);
+ iap.getModificationCallback(), iap.getSearchOperationCallback(), numBloomFilterKeyFields,
+ getTreeFields(), getFilterFields(), getHarness(), getFilterCmpFactories(), tracer);
}
@Override
public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) {
- return createAccessor(createOpContext(iap.getModificationCallback(), iap.getSearchOperationCallback()));
+ return createAccessor(createOpContext(iap));
}
public ILSMIndexAccessor createAccessor(AbstractLSMIndexOperationContext opCtx) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 908a5c2..fef5516 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -38,7 +38,6 @@ import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
-import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -63,10 +62,9 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
-import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.logging.log4j.Level;
@@ -356,8 +354,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
throws HyracksDataException {
LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- AbstractLSMIndexOperationContext opCtx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(ctx.getComponentHolder());
ILSMIOOperation flushOp = createFlushOperation(opCtx, componentFileRefs, callback);
@@ -369,8 +366,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
throws HyracksDataException {
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
// merge must create a different op ctx
- AbstractLSMIndexOperationContext opCtx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(mergingComponents);
ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
@@ -762,8 +758,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
protected abstract LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
ILSMDiskComponent lastComponent) throws HyracksDataException;
- protected abstract AbstractLSMIndexOperationContext createOpContext(
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback)
+ protected abstract AbstractLSMIndexOperationContext createOpContext(IIndexAccessParameters iap)
throws HyracksDataException;
protected abstract ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
index 6f583b4..27966d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInPlaceInvertedIndex.java
@@ -18,18 +18,29 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
public interface IInPlaceInvertedIndex extends IInvertedIndex {
/**
- * Create an inverted list cursor
+ * Creates an inverted list cursor.
+ *
+ * @throws HyracksDataException
+ */
+ InvertedListCursor createInvertedListCursor(IHyracksTaskContext ctx) throws HyracksDataException;
+
+ /**
+ * Creates an inverted-list-range-search cursor. This cursor is mainly used to conduct
+ * a full scan of an inverted list during a storage-component-merge operation.
+ *
+ * @throws HyracksDataException
*/
- IInvertedListCursor createInvertedListCursor();
+ InvertedListCursor createInvertedListRangeSearchCursor() throws HyracksDataException;
/**
- * Open an inverted list cursor
+ * Opens an inverted list cursor
*
* @param listCursor
* the cursor to open
@@ -39,6 +50,6 @@ public interface IInPlaceInvertedIndex extends IInvertedIndex {
* the operation context under which the cursor is to be open
* @throws HyracksDataException
*/
- void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
+ void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx)
throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
index 86c0bf1..fe29c5f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexAccessor.java
@@ -26,12 +26,12 @@ import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
public interface IInvertedIndexAccessor extends IIndexAccessor {
- public IInvertedListCursor createInvertedListCursor();
+ public InvertedListCursor createInvertedListCursor() throws HyracksDataException;
- public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey)
+ public void openInvertedListCursor(InvertedListCursor listCursor, ITupleReference searchKey)
throws HyracksDataException;
- public IIndexCursor createRangeSearchCursor();
+ public IIndexCursor createRangeSearchCursor() throws HyracksDataException;
public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/afe0d3d9/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
index 17e8ad2..837870f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndexSearcher.java
@@ -19,27 +19,36 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.api;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate;
+import org.apache.hyracks.storage.common.IIndexCursor;
+/**
+ * Inverted index search cursor class that conducts the actual inverted index traversal
+ *
+ */
public interface IInvertedIndexSearcher {
- public void search(OnDiskInvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred,
- IIndexOperationContext ictx) throws HyracksDataException;
+ /**
+ * Searches the given inverted index using the search predicate and initializes the result cursor.
+ */
+ public void search(IIndexCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOperationContext ictx)
+ throws HyracksDataException;
- public IFrameTupleAccessor createResultFrameTupleAccessor();
+ /**
+ * Continues the search process if it is paused. (e.g., output buffer full)
+ *
+ * @return true only if all search process is done.
+ * false otherwise.
+ */
+ public boolean continueSearch() throws HyracksDataException;
- public ITupleReference createResultFrameTupleReference();
+ public boolean hasNext() throws HyracksDataException;
- public List<ByteBuffer> getResultBuffers();
+ public void next() throws HyracksDataException;
- public int getNumValidResultBuffers();
+ public void destroy() throws HyracksDataException;
- public void reset();
+ public ITupleReference getTuple();
}