You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/12/09 02:56:07 UTC
Change in asterixdb[master]: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405 )
Change subject: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
......................................................................
[NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
4 files changed, 33 insertions(+), 6 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/14405/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 3e5d5ae..76aeae5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -47,6 +47,7 @@
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.ArrayIndexUtil;
+import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
@@ -2103,7 +2104,7 @@
// an inverted index contains a part of a field value, not all of it.
if (noIndexOnlyPlanOption || dataset.getDatasetType() == DatasetType.EXTERNAL || chosenIndex.isPrimaryIndex()
|| chosenIndex.getIndexDetails().isOverridingKeyFieldTypes() || chosenIndex.isEnforced()
- || isInvertedIndex(chosenIndex) || chosenIndex.getIndexType() == IndexType.ARRAY) {
+ || isInvertedIndex(chosenIndex) || chosenIndex.getIndexType() == IndexType.ARRAY || IndexUtil.includesUnknowns(chosenIndex)) {
indexOnlyPlanInfo.setFirst(false);
return;
}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt b/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
index da02111..1535263 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
@@ -19,3 +19,9 @@
distinct_aggregate.aql
unnest_list_in_subplan.aql
fj-phase2-with-hints.aql
+btree-index/btree-sidx-idxonly-01.sqlpp
+btree-index/btree-sidx-idxonly-01_ps.sqlpp
+btree-index/btree-sidx-idxonly-10.sqlpp
+btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
+btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.sqlpp
+btree-index-selection/btree-index-composite-key-04.sqlpp
\ No newline at end of file
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index ab5b362..ad92803 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -45,6 +45,7 @@
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.job.IJobletEventListenerFactory;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.util.OptionalBoolean;
public class IndexUtil {
@@ -216,4 +217,27 @@
return null;
}
}
+
+ public static boolean includesUnknowns(Index index) {
+ return !index.isPrimaryKeyIndex() && secondaryIndexIncludesUnknowns(index);
+ }
+
+ private static boolean secondaryIndexIncludesUnknowns(Index index) {
+ if (Index.IndexCategory.of(index.getIndexType()) != Index.IndexCategory.VALUE) {
+ // other types of indexes do not include unknowns
+ return false;
+ }
+ OptionalBoolean excludeUnknownKey = ((Index.ValueIndexDetails) index.getIndexDetails()).getExcludeUnknownKey();
+ if (index.getIndexType() == DatasetConfig.IndexType.BTREE) {
+ // by default, Btree includes unknowns
+ return excludeUnknownKey.isEmpty() || !excludeUnknownKey.get();
+ } else {
+ // by default, others exclude unknowns
+ return !excludeUnknownKey.isEmpty() && !excludeUnknownKey.get();
+ }
+ }
+
+ public static boolean excludesUnknowns(Index index) {
+ return !includesUnknowns(index);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 8be248b..9e0d650 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -436,11 +436,7 @@
protected static boolean excludeUnknownKeys(Index index, Index.ValueIndexDetails details,
boolean anySecKeyIsNullable) {
- return excludeUnknowns(index, details) && (anySecKeyIsNullable || details.isOverridingKeyFieldTypes());
- }
-
- private static boolean excludeUnknowns(Index index, Index.ValueIndexDetails details) {
- return index.isPrimaryKeyIndex() || details.getExcludeUnknownKey().getOrElse(false);
+ return IndexUtil.excludesUnknowns(index) && (anySecKeyIsNullable || details.isOverridingKeyFieldTypes());
}
protected LSMIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
Gerrit-Change-Number: 14405
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405 )
Change subject: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
......................................................................
Patch Set 2: Contrib-2
Analytics Compatibility Tests Failed
https://cbjenkins.page.link/Uj3XdQ8yheFSFwpv8 : UNSTABLE
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
Gerrit-Change-Number: 14405
Gerrit-PatchSet: 2
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Thu, 09 Dec 2021 22:40:25 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment
Change in asterixdb[master]: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405 )
Change subject: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
......................................................................
[NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
4 files changed, 33 insertions(+), 6 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/14405/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 3e5d5ae..76aeae5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -47,6 +47,7 @@
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.ArrayIndexUtil;
+import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
@@ -2103,7 +2104,7 @@
// an inverted index contains a part of a field value, not all of it.
if (noIndexOnlyPlanOption || dataset.getDatasetType() == DatasetType.EXTERNAL || chosenIndex.isPrimaryIndex()
|| chosenIndex.getIndexDetails().isOverridingKeyFieldTypes() || chosenIndex.isEnforced()
- || isInvertedIndex(chosenIndex) || chosenIndex.getIndexType() == IndexType.ARRAY) {
+ || isInvertedIndex(chosenIndex) || chosenIndex.getIndexType() == IndexType.ARRAY || IndexUtil.includesUnknowns(chosenIndex)) {
indexOnlyPlanInfo.setFirst(false);
return;
}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt b/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
index da02111..1535263 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
@@ -19,3 +19,9 @@
distinct_aggregate.aql
unnest_list_in_subplan.aql
fj-phase2-with-hints.aql
+btree-index/btree-sidx-idxonly-01.sqlpp
+btree-index/btree-sidx-idxonly-01_ps.sqlpp
+btree-index/btree-sidx-idxonly-10.sqlpp
+btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
+btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_03-index-only.sqlpp
+btree-index-selection/btree-index-composite-key-04.sqlpp
\ No newline at end of file
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index ab5b362..ad92803 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -45,6 +45,7 @@
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.job.IJobletEventListenerFactory;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.util.OptionalBoolean;
public class IndexUtil {
@@ -216,4 +217,27 @@
return null;
}
}
+
+ public static boolean includesUnknowns(Index index) {
+ return !index.isPrimaryKeyIndex() && secondaryIndexIncludesUnknowns(index);
+ }
+
+ private static boolean secondaryIndexIncludesUnknowns(Index index) {
+ if (Index.IndexCategory.of(index.getIndexType()) != Index.IndexCategory.VALUE) {
+ // other types of indexes do not include unknowns
+ return false;
+ }
+ OptionalBoolean excludeUnknownKey = ((Index.ValueIndexDetails) index.getIndexDetails()).getExcludeUnknownKey();
+ if (index.getIndexType() == DatasetConfig.IndexType.BTREE) {
+ // by default, Btree includes unknowns
+ return excludeUnknownKey.isEmpty() || !excludeUnknownKey.get();
+ } else {
+ // by default, others exclude unknowns
+ return !excludeUnknownKey.isEmpty() && !excludeUnknownKey.get();
+ }
+ }
+
+ public static boolean excludesUnknowns(Index index) {
+ return !includesUnknowns(index);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 8be248b..9e0d650 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -436,11 +436,7 @@
protected static boolean excludeUnknownKeys(Index index, Index.ValueIndexDetails details,
boolean anySecKeyIsNullable) {
- return excludeUnknowns(index, details) && (anySecKeyIsNullable || details.isOverridingKeyFieldTypes());
- }
-
- private static boolean excludeUnknowns(Index index, Index.ValueIndexDetails details) {
- return index.isPrimaryKeyIndex() || details.getExcludeUnknownKey().getOrElse(false);
+ return IndexUtil.excludesUnknowns(index) && (anySecKeyIsNullable || details.isOverridingKeyFieldTypes());
}
protected LSMIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
Gerrit-Change-Number: 14405
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Jenkins <je...@fulliautomatix.ics.uci.edu>:
Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405 )
Change subject: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
......................................................................
Patch Set 1: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/12804/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
Gerrit-Change-Number: 14405
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-CC: Anon. E. Moose #1000171
Gerrit-Comment-Date: Thu, 09 Dec 2021 04:22:53 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment
Change in asterixdb[master]: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Ali Alsuliman <al...@gmail.com>:
Hello Jenkins, Anon. E. Moose #1000171,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
to look at the new patch set (#2).
Change subject: [NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
......................................................................
[NO ISSUE][IDX] Disable index only plan when index includes UNKNOWNs
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M asterixdb/asterix-app/src/test/resources/optimizerts/ignore.txt
M asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/secondary-index-index-only/secondary-index-index-only.3.regexadm
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
5 files changed, 35 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/14405/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14405
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I330aaaa1792ad7fe1b264f6e10577c3defe8eeec
Gerrit-Change-Number: 14405
Gerrit-PatchSet: 2
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-MessageType: newpatchset