You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Ali Alsuliman (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/12/19 22:22:15 UTC
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Ali Alsuliman has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2240
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
- user model changes: no
- storage format changes: no
- interface changes: no
details:
This change is intended to remove the sort operator when creating
a secondary primary index. The secondary primary index DDL statement uses
the same code as the one for creating general secondary BTree indexes which
has the sort operator since it needs to sort by SK. The sort operator is
not needed for secondary primary index since there is no SK and PK is already
sorted.
Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
---
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
1 file changed, 39 insertions(+), 32 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/40/2240/1
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 41def96..cf6338a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -125,50 +125,57 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
} else {
- // Create dummy key provider for feeding the primary index scan.
- IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset,
- metadataProvider);
+ // job spec:
+ // key provider -> primary idx -> (cast assign)? -> assign -> (select)? -> (sort)? -> bulk load -> sink
IndexUtil.bindJobEventListener(spec, metadataProvider);
- // Create primary index scan op.
- IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ // dummy key provider ----> primary index scan
+ IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+ IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // Assign op.
- IOperatorDescriptor sourceOp = primaryScanOp;
+ sourceOp = targetOp;
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
- sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ // primary index scan ----> cast assign
+ targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp =
- createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // primary index OR cast assign ----> assign op
+ targetOp = createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // If any of the secondary fields are nullable, then add a select op that filters nulls.
- AlgebricksMetaOperatorDescriptor selectOp = null;
+ sourceOp = targetOp;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // if any of the secondary fields are nullable, then add a select op that filters nulls.
+ // assign op ----> select op
+ targetOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- // Sort by secondary keys.
- ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
- // Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation,
- dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ // no need to sort if the index is secondary primary index
+ if (!index.getKeyFieldNames().isEmpty()) {
+ // sort by secondary keys.
+ // assign op OR select op ----> sort op
+ targetOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
+ }
- AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ // assign op OR select op OR sort op ----> bulk load op
+ targetOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory,
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ // bulk load op ----> sink op
+ sourceOp = targetOp;
+ targetOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
new RecordDescriptor[] { secondaryRecDesc });
- // Connect the operators.
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
- if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
- } else {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
- }
- spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
- spec.addRoot(metaOp);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ spec.addRoot(targetOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Anon. E. Moose (Code Review)" <do...@asterixdb.incubator.apache.org>.
Anon. E. Moose #1000171 has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: Contrib+1
Analytics Compatibility Tests Successful
https://goo.gl/kYqX8r : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/2850/ (12/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app-sql-execution/2003/ (1/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-rebalance-cancellation/1190/ (6/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-source-assemblies/2300/ (8/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/2847/ (4/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/2349/
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-asterix-app/2631/ (5/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-cancellation-test/1985/ (7/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5005/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5002/
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-sonar/6021/ (3/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/hyracks-gerrit/1969/ (9/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Dmitry Lychagin (Code Review)" <do...@asterixdb.incubator.apache.org>.
Dmitry Lychagin has submitted this change and it was merged.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
- user model changes: no
- storage format changes: no
- interface changes: no
details:
This change is intended to remove the sort operator when creating
a secondary primary index. The secondary primary index DDL statement uses
the same code as the one for creating general secondary BTree indexes which
has the sort operator since it needs to sort by SK. The sort operator is
not needed for secondary primary index since there is no SK and PK is already
sorted.
Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2240
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
1 file changed, 39 insertions(+), 32 deletions(-)
Approvals:
Anon. E. Moose #1000171:
Jenkins: Verified; No violations found; ; Verified
Dmitry Lychagin: Looks good to me, approved
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 41def96..cf6338a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -125,50 +125,57 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
} else {
- // Create dummy key provider for feeding the primary index scan.
- IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset,
- metadataProvider);
+ // job spec:
+ // key provider -> primary idx -> (cast assign)? -> assign -> (select)? -> (sort)? -> bulk load -> sink
IndexUtil.bindJobEventListener(spec, metadataProvider);
- // Create primary index scan op.
- IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ // dummy key provider ----> primary index scan
+ IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+ IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // Assign op.
- IOperatorDescriptor sourceOp = primaryScanOp;
+ sourceOp = targetOp;
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
- sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ // primary index scan ----> cast assign
+ targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp =
- createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // primary index OR cast assign ----> assign op
+ targetOp = createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // If any of the secondary fields are nullable, then add a select op that filters nulls.
- AlgebricksMetaOperatorDescriptor selectOp = null;
+ sourceOp = targetOp;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // if any of the secondary fields are nullable, then add a select op that filters nulls.
+ // assign op ----> select op
+ targetOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- // Sort by secondary keys.
- ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
- // Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation,
- dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ // no need to sort if the index is secondary primary index
+ if (!index.getKeyFieldNames().isEmpty()) {
+ // sort by secondary keys.
+ // assign op OR select op ----> sort op
+ targetOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
+ }
- AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ // assign op OR select op OR sort op ----> bulk load op
+ targetOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory,
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ // bulk load op ----> sink op
+ sourceOp = targetOp;
+ targetOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
new RecordDescriptor[] { secondaryRecDesc });
- // Connect the operators.
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
- if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
- } else {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
- }
- spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
- spec.addRoot(metaOp);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ spec.addRoot(targetOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Dmitry Lychagin (Code Review)" <do...@asterixdb.incubator.apache.org>.
Dmitry Lychagin has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Dmitry Lychagin <dm...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-rebalance-cancellation/1193/ (11/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/2852/ (12/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/7522/ (10/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-no-installer-app/2619/ (2/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-verify-storage/2598/ (11/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-ensure-ancestor/78/ (12/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: Contrib+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/2349/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: Integration-Tests-1
Integration Tests Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5002/ : UNSTABLE
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary ...
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
......................................................................
Patch Set 1: -Integration-Tests
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5005/
--
To view, visit https://asterix-gerrit.ics.uci.edu/2240
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No