You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2018/10/16 04:18:51 UTC
[21/21] asterixdb git commit: [ASTERIXDB-2286][COMP][FUN][HYR]
Parallel Sort Optimization
[ASTERIXDB-2286][COMP][FUN][HYR] Parallel Sort Optimization
- user model changes: yes
- storage format changes: no
- interface changes: yes
details:
- new plan for sort operation which includes sampling and
replicating the stream of data to be sorted. Sort-merge connector
is removed from the plan. The sorted result now is in multiple partitions.
- new optimization rule to check whether full parallel sort is applicable.
- new Forward operator to read the replicated sort input stream and
to receive the ouput of the sampling.
- new sequential merge connector to merge a globally ordered result residing
in multiple partitions (in addition to the connector's partition computer).
- "asterix-lang-aql/pom.xml" is changed as a result of refactoring
code related to the range map handling.
- new private sampling function to generate the range map object
(local & global functions) & their type computers.
user model changes:
- new compiler property is added to enable and disable parallel sort.
interface changes:
- "ILogicalOperatorVisitor.java" includes Forward Operator.
- "ITuplePartitionComputer.java" includes initialize() to enable partitioner
to do some initialization. FieldRangePartitionComputerFactory uses it to
pick a range map.
- "ITuplePartitionComputerFactory.java". createPartitioner() is changed to
createPartitioner(IHyracksTaskContext hyracksTaskContext). Context is needed
for transferring the range map throught the context.
Change-Id: I73e128029a46f45e6b68c23dfb9310d5de10582f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2393
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/80225e2c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/80225e2c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/80225e2c
Branch: refs/heads/master
Commit: 80225e2c27d77514ecaa774235951187ef524193
Parents: adfb633
Author: Ali Alsuliman <al...@gmail.com>
Authored: Mon Oct 15 14:17:07 2018 -0700
Committer: Ali Alsuliman <al...@gmail.com>
Committed: Mon Oct 15 21:17:44 2018 -0700
----------------------------------------------------------------------
.../asterix/optimizer/base/RuleCollections.java | 7 +-
.../rules/CheckFullParallelSortRule.java | 79 +
.../rules/FixReplicateOperatorOutputsRule.java | 2 +-
.../SweepIllegalNonfunctionalFunctions.java | 19 +-
.../subplan/InlineAllNtsInSubplanVisitor.java | 7 +
...neLeftNtsInSubplanJoinFlatteningVisitor.java | 35 +-
.../SubplanSpecialFlatteningCheckVisitor.java | 11 +-
.../LangExpressionToPlanTranslator.java | 6 +-
.../apache/asterix/api/common/APIFramework.java | 30 +-
.../app/resource/OperatorResourcesComputer.java | 1 +
.../app/resource/PlanStagesGenerator.java | 90 +-
.../app/resource/RequiredCapacityVisitor.java | 7 +
.../app/translator/RequestParameters.java | 2 +-
.../asterix-app/src/main/resources/cc.conf | 1 +
.../dataflow/TestPartitionComputerFactory.java | 3 +-
.../asterix/test/optimizer/OptimizerTest.java | 5 +
...-probe-pidx-with-join-btree-sidx_01_ps.sqlpp | 76 +
...-probe-pidx-with-join-btree-sidx_02_ps.sqlpp | 78 +
.../btree-index/btree-secondary-68_ps.sqlpp | 56 +
...ree-sidx-idxonly-01-disable-idxonly_ps.sqlpp | 68 +
.../btree-index/btree-sidx-idxonly-01_ps.sqlpp | 66 +
.../queries/introhashpartitionmerge.sqlpp | 6 +-
.../ngram-contains-panic_ps.sqlpp | 51 +
.../ngram-contains_ps.sqlpp | 51 +
.../olist-edit-distance-check-panic_ps.sqlpp | 63 +
.../olist-edit-distance-check_ps.sqlpp | 63 +
.../olist-edit-distance-panic_ps.sqlpp | 63 +
.../olist-edit-distance_ps.sqlpp | 63 +
.../olist-fuzzyeq-edit-distance_ps.sqlpp | 65 +
.../inverted-index-basic/word-contains_ps.sqlpp | 51 +
...olist-edit-distance-check-let-panic_ps.sqlpp | 65 +
.../olist-edit-distance-check-let_ps.sqlpp | 65 +
...ith-join-edit-distance-check-idx_01_ps.sqlpp | 81 +
...pidx-with-join-jaccard-check-idx_01_ps.sqlpp | 81 +
.../inverted-index-join/ngram-contains_ps.sqlpp | 52 +
.../optimizerts/queries/loj-core_ps.sqlpp | 52 +
.../optimizerts/queries/loj-sugar_ps.sqlpp | 48 +
...-probe-pidx-with-join-btree-sidx_01_ps.sqlpp | 75 +
...-probe-pidx-with-join-btree-sidx_02_ps.sqlpp | 75 +
.../ngram-contains-panic_ps.sqlpp | 56 +
.../ngram-contains_ps.sqlpp | 56 +
.../olist-edit-distance-check-panic_ps.sqlpp | 70 +
.../olist-edit-distance-check_ps.sqlpp | 70 +
.../olist-edit-distance-panic_ps.sqlpp | 70 +
.../olist-edit-distance_ps.sqlpp | 70 +
.../olist-fuzzyeq-edit-distance_ps.sqlpp | 70 +
.../inverted-index-basic/word-contains_ps.sqlpp | 56 +
...olist-edit-distance-check-let-panic_ps.sqlpp | 72 +
.../olist-edit-distance-check-let_ps.sqlpp | 72 +
...ith-join-edit-distance-check-idx_01_ps.sqlpp | 76 +
...pidx-with-join-jaccard-check-idx_01_ps.sqlpp | 76 +
...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp | 82 +
...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp | 82 +
...robe-pidx-with-join-btree-sidx_01_1_ps.sqlpp | 74 +
...robe-pidx-with-join-btree-sidx_01_2_ps.sqlpp | 75 +
...robe-pidx-with-join-btree-sidx_02_1_ps.sqlpp | 74 +
...robe-pidx-with-join-btree-sidx_02_2_ps.sqlpp | 75 +
.../ngram-contains-panic_ps.sqlpp | 55 +
.../ngram-contains_ps.sqlpp | 55 +
.../inverted-index-basic/word-contains_ps.sqlpp | 55 +
...ith-join-edit-distance-check-idx_01_ps.sqlpp | 75 +
.../ngram-contains_01_ps.sqlpp | 61 +
.../ngram-contains_02_ps.sqlpp | 61 +
.../ngram-contains_03_ps.sqlpp | 51 +
.../ngram-contains_04_ps.sqlpp | 62 +
...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp | 81 +
...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp | 81 +
...robe-pidx-with-join-btree-sidx_01_1_ps.sqlpp | 69 +
...robe-pidx-with-join-btree-sidx_01_2_ps.sqlpp | 70 +
...robe-pidx-with-join-btree-sidx_02_1_ps.sqlpp | 69 +
...robe-pidx-with-join-btree-sidx_02_2_ps.sqlpp | 70 +
.../ngram-contains-panic_ps.sqlpp | 50 +
.../ngram-contains_ps.sqlpp | 50 +
.../inverted-index-basic/word-contains_ps.sqlpp | 50 +
...ith-join-edit-distance-check-idx_01_ps.sqlpp | 70 +
.../ngram-contains_01_ps.sqlpp | 61 +
.../ngram-contains_02_ps.sqlpp | 61 +
.../ngram-contains_03_ps.sqlpp | 51 +
.../ngram-contains_04_ps.sqlpp | 62 +
...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp | 76 +
...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp | 76 +
.../btree-equi-join-non-enforced-05_ps.sqlpp | 46 +
.../btree-equi-join-non-enforced-06_ps.sqlpp | 46 +
.../btree-equi-join-non-enforced-07_ps.sqlpp | 47 +
.../btree-equi-join-non-enforced-08_ps.sqlpp | 47 +
.../btree-equi-join-non-enforced-09_ps.sqlpp | 46 +
.../btree-index-non-enforced-04_ps.sqlpp | 44 +
.../btree-index-non-enforced-05_ps.sqlpp | 44 +
.../btree-index-non-enforced-06_ps.sqlpp | 44 +
.../btree-index-non-enforced-07_ps.sqlpp | 44 +
.../btree-index-non-enforced-08_ps.sqlpp | 44 +
.../btree-index-non-enforced-09_ps.sqlpp | 44 +
.../btree-index-non-enforced-105_ps.sqlpp | 46 +
.../btree-index-non-enforced-10_ps.sqlpp | 44 +
.../btree-index-non-enforced-11_ps.sqlpp | 44 +
.../queries/orderby-desc-using-gby_ps.sqlpp | 59 +
...rs-index-search-conjunctive-open_01_ps.sqlpp | 56 +
...rs-index-search-conjunctive-open_02_ps.sqlpp | 56 +
.../orders-index-search-conjunctive_01_ps.sqlpp | 56 +
.../orders-index-search-conjunctive_02_ps.sqlpp | 56 +
.../parallel_sort_enabled_disabled.1.sqlpp | 40 +
.../parallel_sort_enabled_disabled.2.sqlpp | 40 +
.../q01_pricing_summary_report_nt_ps.sqlpp | 80 +
.../queries/query-ASTERIXDB-1806_ps.sqlpp | 66 +
.../queries/query-ASTERIXDB-2354_ps.sqlpp | 29 +
.../queries/query-ASTERIXDB-2408_ps.sqlpp | 45 +
.../queries/query-ASTERIXDB-810-2_ps.sqlpp | 70 +
.../queries/query-ASTERIXDB-810-3_ps.sqlpp | 70 +
.../queries/query-ASTERIXDB-810_ps.sqlpp | 70 +
.../optimizerts/queries/query-issue562_ps.sqlpp | 108 +
...-probe-pidx-with-join-rtree-sidx_01_ps.sqlpp | 77 +
...-probe-pidx-with-join-rtree-sidx_02_ps.sqlpp | 77 +
.../skip-index/skip-ngram-index_ps.sqlpp | 51 +
.../queries/split-materialization_ps.sqlpp | 61 +
.../queries/subquery/exists_ps.sqlpp | 72 +
.../queries/subquery/in_as_or_1_ps.sqlpp | 42 +
.../queries/subquery/in_as_or_2_ps.sqlpp | 43 +
.../queries/subquery/in_as_or_3_ps.sqlpp | 46 +
.../queries/subquery/in_as_or_4_ps.sqlpp | 47 +
.../queries/subquery/in_as_or_5_ps.sqlpp | 48 +
.../queries/subquery/in_as_or_6_ps.sqlpp | 49 +
.../queries/subquery/in_correlated_ps.sqlpp | 44 +
.../optimizerts/queries/subquery/in_ps.sqlpp | 44 +
.../queries/subquery/not_exists_ps.sqlpp | 73 +
.../queries/tpcds/query-ASTERIXDB-1596_ps.sqlpp | 131 +
.../tpch/q12_shipping_broadcast_ps.sqlpp | 53 +
.../queries/tpch/q12_shipping_ps.sqlpp | 53 +
...n-probe-pidx-with-join-btree-sidx_01_ps.plan | 109 +
...n-probe-pidx-with-join-btree-sidx_02_ps.plan | 77 +
.../secondary-self-equi-join-index-only.plan | 16 +-
...on-idxonly-to-sidx-idxonly-equi-join_01.plan | 50 +-
.../btree-index/btree-secondary-68_ps.plan | 69 +
...tree-sidx-idxonly-01-disable-idxonly_ps.plan | 47 +
.../btree-index/btree-sidx-idxonly-01_ps.plan | 65 +
.../results/hints/fullparallelsort.plan | 117 +
.../results/hints/nofullparallel_hint.plan | 38 +
.../ngram-contains-panic_ps.plan | 28 +
.../inverted-index-basic/ngram-contains_ps.plan | 38 +
.../olist-edit-distance-check-panic_ps.plan | 28 +
.../olist-edit-distance-check_ps.plan | 38 +
.../olist-edit-distance-panic_ps.plan | 28 +
.../olist-edit-distance_ps.plan | 38 +
.../olist-fuzzyeq-edit-distance_ps.plan | 38 +
.../inverted-index-basic/word-contains_ps.plan | 28 +
.../olist-edit-distance-check-let-panic_ps.plan | 28 +
.../olist-edit-distance-check-let_ps.plan | 38 +
...with-join-edit-distance-check-idx_01_ps.plan | 165 +
...-pidx-with-join-jaccard-check-idx_01_ps.plan | 353 +
.../inverted-index-join/ngram-contains_ps.plan | 54 +
.../optimizerts/results/loj-core_ps.plan | 53 +
.../optimizerts/results/loj-sugar_ps.plan | 53 +
...n-probe-pidx-with-join-btree-sidx_01_ps.plan | 86 +
...n-probe-pidx-with-join-btree-sidx_02_ps.plan | 86 +
.../ngram-contains-panic_ps.plan | 32 +
.../inverted-index-basic/ngram-contains_ps.plan | 42 +
.../olist-edit-distance-check-panic_ps.plan | 32 +
.../olist-edit-distance-check_ps.plan | 42 +
.../olist-edit-distance-panic_ps.plan | 32 +
.../olist-edit-distance_ps.plan | 42 +
.../olist-fuzzyeq-edit-distance_ps.plan | 42 +
.../inverted-index-basic/word-contains_ps.plan | 32 +
.../olist-edit-distance-check-let-panic_ps.plan | 32 +
.../olist-edit-distance-check-let_ps.plan | 42 +
...with-join-edit-distance-check-idx_01_ps.plan | 190 +
...-pidx-with-join-jaccard-check-idx_01_ps.plan | 358 +
...n-probe-pidx-with-join-rtree-sidx_01_ps.plan | 90 +
...n-probe-pidx-with-join-rtree-sidx_02_ps.plan | 92 +
...probe-pidx-with-join-btree-sidx_01_1_ps.plan | 86 +
...probe-pidx-with-join-btree-sidx_01_2_ps.plan | 86 +
...probe-pidx-with-join-btree-sidx_02_1_ps.plan | 86 +
...probe-pidx-with-join-btree-sidx_02_2_ps.plan | 86 +
.../ngram-contains-panic_ps.plan | 32 +
.../inverted-index-basic/ngram-contains_ps.plan | 42 +
.../inverted-index-basic/word-contains_ps.plan | 32 +
...with-join-edit-distance-check-idx_01_ps.plan | 190 +
.../ngram-contains_01_ps.plan | 50 +
.../ngram-contains_02_ps.plan | 50 +
.../ngram-contains_03_ps.plan | 54 +
.../ngram-contains_04_ps.plan | 54 +
...n-probe-pidx-with-join-rtree-sidx_01_ps.plan | 90 +
...n-probe-pidx-with-join-rtree-sidx_02_ps.plan | 92 +
...probe-pidx-with-join-btree-sidx_01_1_ps.plan | 77 +
...probe-pidx-with-join-btree-sidx_01_2_ps.plan | 77 +
...probe-pidx-with-join-btree-sidx_02_1_ps.plan | 77 +
...probe-pidx-with-join-btree-sidx_02_2_ps.plan | 77 +
.../ngram-contains-panic_ps.plan | 28 +
.../inverted-index-basic/ngram-contains_ps.plan | 38 +
.../inverted-index-basic/word-contains_ps.plan | 28 +
...with-join-edit-distance-check-idx_01_ps.plan | 165 +
.../ngram-contains_01_ps.plan | 50 +
.../ngram-contains_02_ps.plan | 54 +
.../ngram-contains_03_ps.plan | 54 +
.../ngram-contains_04_ps.plan | 54 +
...n-probe-pidx-with-join-rtree-sidx_01_ps.plan | 83 +
...n-probe-pidx-with-join-rtree-sidx_02_ps.plan | 83 +
.../btree-equi-join-non-enforced-05_ps.plan | 56 +
.../btree-equi-join-non-enforced-06_ps.plan | 56 +
.../btree-equi-join-non-enforced-07_ps.plan | 56 +
.../btree-equi-join-non-enforced-08_ps.plan | 58 +
.../btree-equi-join-non-enforced-09_ps.plan | 58 +
.../btree-index-non-enforced-04_ps.plan | 48 +
.../btree-index-non-enforced-05_ps.plan | 48 +
.../btree-index-non-enforced-06_ps.plan | 48 +
.../btree-index-non-enforced-07_ps.plan | 48 +
.../btree-index-non-enforced-08_ps.plan | 48 +
.../btree-index-non-enforced-09_ps.plan | 48 +
.../btree-index-non-enforced-105_ps.plan | 70 +
.../btree-index-non-enforced-10_ps.plan | 48 +
.../btree-index-non-enforced-11_ps.plan | 50 +
.../results/orderby-desc-using-gby_ps.plan | 44 +
...ers-index-search-conjunctive-open_01_ps.plan | 60 +
...ers-index-search-conjunctive-open_02_ps.plan | 60 +
.../orders-index-search-conjunctive_01_ps.plan | 60 +
.../orders-index-search-conjunctive_02_ps.plan | 60 +
.../parallel_sort_enabled_disabled.1.plan | 30 +
.../parallel_sort_enabled_disabled.2.plan | 12 +
.../q01_pricing_summary_report_nt_ps.plan | 74 +
.../results/query-ASTERIXDB-1806_ps.plan | 59 +
.../results/query-ASTERIXDB-2354_ps.plan | 98 +
.../results/query-ASTERIXDB-2408_ps.plan | 72 +
.../results/query-ASTERIXDB-810-2_ps.plan | 83 +
.../results/query-ASTERIXDB-810-3_ps.plan | 83 +
.../results/query-ASTERIXDB-810_ps.plan | 83 +
.../optimizerts/results/query-issue562_ps.plan | 135 +
.../rtree-index-join/issue730-index-only.plan | 22 +-
...n-probe-pidx-with-join-rtree-sidx_01_ps.plan | 133 +
...n-probe-pidx-with-join-rtree-sidx_02_ps.plan | 83 +
...spatial-self-intersect-point-index-only.plan | 16 +-
.../results/skip-index/skip-ngram-index_ps.plan | 28 +
.../results/split-materialization_ps.plan | 67 +
.../optimizerts/results/subquery/exists_ps.plan | 157 +
.../results/subquery/in_as_or_1_ps.plan | 41 +
.../results/subquery/in_as_or_2_ps.plan | 67 +
.../results/subquery/in_as_or_3_ps.plan | 41 +
.../results/subquery/in_as_or_4_ps.plan | 67 +
.../results/subquery/in_as_or_5_ps.plan | 41 +
.../results/subquery/in_as_or_6_ps.plan | 67 +
.../results/subquery/in_correlated_ps.plan | 84 +
.../optimizerts/results/subquery/in_ps.plan | 74 +
.../results/subquery/not_exists_ps.plan | 157 +
.../results/tpcds/query-ASTERIXDB-1596_ps.plan | 58 +
.../results/tpch/q12_shipping_broadcast_ps.plan | 95 +
.../results/tpch/q12_shipping_ps.plan | 95 +
.../big_object_insert.3.query.aql | 4 +-
.../prefix-search/prefix-search.3.query.aql | 2 +-
.../misc/stable_sort/stable_sort.3.query.aql | 2 +-
.../overlap_bins_gby_0.3.query.aql | 4 +-
.../p_sort_num_samples.1.ddl.sqlpp | 47 +
.../p_sort_num_samples.2.update.sqlpp | 22 +
.../p_sort_num_samples.3.query.sqlpp | 26 +
.../p_sort_seq_merge.1.ddl.sqlpp | 36 +
.../p_sort_seq_merge.2.update.sqlpp | 42 +
.../p_sort_seq_merge.3.query.sqlpp | 24 +
.../prefix-search/prefix-search.3.query.sqlpp | 2 +-
.../misc/stable_sort/stable_sort.3.query.sqlpp | 3 +-
.../object_remove/object_remove.3.query.sqlpp | 2 +-
.../single_dataset_with_index.13.query.sqlpp | 2 +
.../single_dataset_with_index.8.query.sqlpp | 2 +
.../overlap_bins_gby_0.3.query.sqlpp | 2 +-
.../cluster_state_1/cluster_state_1.1.regexadm | 2 +
.../cluster_state_1_full.1.regexadm | 2 +
.../cluster_state_1_less.1.regexadm | 2 +
.../p_sort_num_samples/p_sort_num_samples.3.adm | 6005 ++++++++++++++++
.../p_sort_seq_merge/p_sort_seq_merge.3.adm | 1 +
.../misc/prefix-search/prefix-search.1.adm | 90 +-
.../results/misc/stable_sort/stable_sort.3.adm | 6638 +++++++++---------
.../misc/prefix-search/prefix-search.3.ast | 5 +
.../misc/stable_sort/stable_sort.3.ast | 10 +
.../overlap_bins_gby_0/overlap_bins_gby_0.3.ast | 10 +
.../resources/runtimets/testsuite_sqlpp.xml | 10 +
.../common/config/CompilerProperties.java | 21 +-
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../main/resources/asx_errormsg/en.properties | 1 +
.../ByteArrayAccessibleDataInputStream.java | 33 -
.../base/ByteArrayAccessibleInputStream.java | 48 -
asterixdb/asterix-lang-aql/pom.xml | 8 -
.../asterix/lang/aql/util/RangeMapBuilder.java | 177 -
.../asterix-lang-aql/src/main/javacc/AQL.jj | 9 +-
asterixdb/asterix-lang-common/pom.xml | 4 +
.../lang/common/clause/OrderbyClause.java | 13 +-
.../lang/common/util/RangeMapBuilder.java | 175 +
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 14 +-
.../asterix/om/functions/BuiltinFunctions.java | 15 +
.../impl/ListOfSamplesTypeComputer.java | 45 +
.../std/LocalSamplingAggregateDescriptor.java | 173 +
.../std/RangeMapAggregateDescriptor.java | 317 +
.../runtime/functions/FunctionCollection.java | 4 +
.../runtime/functions/FunctionTypeInferers.java | 26 +-
.../core/algebra/base/LogicalOperatorTag.java | 1 +
.../core/algebra/base/OperatorAnnotations.java | 3 +-
.../core/algebra/base/PhysicalOperatorTag.java | 2 +
.../AbstractFunctionCallExpression.java | 19 +-
.../AggregateFunctionCallExpression.java | 2 +
.../logical/AbstractReplicateOperator.java | 16 +-
.../operators/logical/ForwardOperator.java | 111 +
.../visitors/CardinalityInferenceVisitor.java | 8 +-
.../visitors/FDsAndEquivClassesVisitor.java | 7 +
.../visitors/IsomorphismOperatorVisitor.java | 13 +
.../IsomorphismVariableMappingVisitor.java | 7 +
...OperatorDeepCopyWithNewVariablesVisitor.java | 9 +
.../visitors/LogicalPropertiesVisitor.java | 9 +-
.../visitors/OperatorDeepCopyVisitor.java | 6 +
.../visitors/PrimaryKeyVariablesVisitor.java | 8 +-
.../visitors/ProducedVariableVisitor.java | 8 +-
.../logical/visitors/SchemaVariableVisitor.java | 8 +
.../visitors/SubstituteVariableVisitor.java | 9 +
.../logical/visitors/UsedVariableVisitor.java | 54 +-
.../physical/AbstractStableSortPOperator.java | 54 +-
.../operators/physical/ForwardPOperator.java | 136 +
.../RangePartitionExchangePOperator.java | 55 +-
.../RangePartitionMergeExchangePOperator.java | 10 +-
.../SequentialMergeExchangePOperator.java | 76 +
.../LogicalOperatorPrettyPrintVisitor.java | 8 +
.../LogicalOperatorPrettyPrintVisitorJson.java | 9 +
.../algebra/properties/LocalOrderProperty.java | 15 +-
.../core/algebra/properties/PropertiesUtil.java | 61 +-
.../visitors/ILogicalOperatorVisitor.java | 2 +
.../core/config/AlgebricksConfig.java | 2 +
.../impl/ConnectorPolicyAssignmentPolicy.java | 4 +-
.../core/rewriter/base/HeuristicOptimizer.java | 37 +-
.../base/PhysicalOptimizationConfig.java | 33 +-
.../core/utils/DotFormatGenerator.java | 152 +-
.../core/utils/LogicalOperatorDotVisitor.java | 390 +-
.../rules/EnforceStructuralPropertiesRule.java | 370 +-
.../rules/ExtractCommonOperatorsRule.java | 66 +
.../rewriter/rules/InlineVariablesRule.java | 30 +-
.../SetAlgebricksPhysicalOperatorsRule.java | 4 +
...placeNtsWithSubplanInputOperatorVisitor.java | 8 +-
.../dataflow/value/ITuplePartitionComputer.java | 18 +
.../value/ITuplePartitionComputerFactory.java | 4 +-
.../hyracks/api/exceptions/ErrorCode.java | 3 +
.../src/main/resources/errormsg/en.properties | 3 +
.../ByteArrayAccessibleDataInputStream.java | 33 +
.../util/ByteArrayAccessibleInputStream.java | 48 +
.../FieldHashPartitionComputerFactory.java | 3 +-
.../partition/OnePartitionComputerFactory.java | 38 +
.../RandomPartitionComputerFactory.java | 3 +-
.../partition/RepartitionComputerFactory.java | 5 +-
...namicFieldRangePartitionComputerFactory.java | 48 +
.../FieldRangePartitionComputerFactory.java | 44 +-
.../common/data/partition/range/IRangeMap.java | 35 -
.../common/data/partition/range/RangeMap.java | 116 +-
...taticFieldRangePartitionComputerFactory.java | 38 +
.../base/AbstractMToNConnectorDescriptor.java | 18 +
.../AbstractReplicateOperatorDescriptor.java | 1 +
.../DeterministicPartitionBatchManager.java | 80 +
.../collectors/SequentialMergeFrameReader.java | 77 +
...wareMToNPartitioningConnectorDescriptor.java | 2 +-
.../MToNBroadcastConnectorDescriptor.java | 16 -
.../MToNPartitioningConnectorDescriptor.java | 19 +-
...NPartitioningMergingConnectorDescriptor.java | 2 +-
...titioningWithMessageConnectorDescriptor.java | 2 +-
...OneSequentialMergingConnectorDescriptor.java | 65 +
.../std/connectors/PartitionDataWriter.java | 1 +
.../join/HybridHashJoinOperatorDescriptor.java | 12 +-
.../InMemoryHashJoinOperatorDescriptor.java | 4 +-
.../std/misc/ForwardOperatorDescriptor.java | 246 +
.../std/sort/AbstractExternalSortRunMerger.java | 9 +-
358 files changed, 27634 insertions(+), 4310 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 3c981d4..1010a84 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -24,6 +24,7 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.rules.AddEquivalenceClassForRecordConstructorRule;
import org.apache.asterix.optimizer.rules.AsterixExtractFunctionsFromJoinConditionRule;
import org.apache.asterix.optimizer.rules.AsterixInlineVariablesRule;
@@ -31,6 +32,7 @@ import org.apache.asterix.optimizer.rules.AsterixIntroduceGroupByCombinerRule;
import org.apache.asterix.optimizer.rules.ByNameToByIndexFieldAccessRule;
import org.apache.asterix.optimizer.rules.CancelUnnestWithNestedListifyRule;
import org.apache.asterix.optimizer.rules.CheckFilterExpressionTypeRule;
+import org.apache.asterix.optimizer.rules.CheckFullParallelSortRule;
import org.apache.asterix.optimizer.rules.CheckInsertUpsertReturningRule;
import org.apache.asterix.optimizer.rules.ConstantFoldingRule;
import org.apache.asterix.optimizer.rules.CountVarToCountOneRule;
@@ -338,7 +340,9 @@ public final class RuleCollections {
physicalRewritesAllLevels.add(new SetAlgebricksPhysicalOperatorsRule());
physicalRewritesAllLevels.add(new SetAsterixPhysicalOperatorsRule());
physicalRewritesAllLevels.add(new AddEquivalenceClassForRecordConstructorRule());
- physicalRewritesAllLevels.add(new EnforceStructuralPropertiesRule());
+ physicalRewritesAllLevels.add(new CheckFullParallelSortRule());
+ physicalRewritesAllLevels
+ .add(new EnforceStructuralPropertiesRule(BuiltinFunctions.RANGE_MAP, BuiltinFunctions.LOCAL_SAMPLING));
physicalRewritesAllLevels.add(new RemoveSortInFeedIngestionRule());
physicalRewritesAllLevels.add(new RemoveUnnecessarySortMergeExchange());
physicalRewritesAllLevels.add(new PushProjectDownRule());
@@ -377,6 +381,7 @@ public final class RuleCollections {
prepareForJobGenRewrites.add(new SetAlgebricksPhysicalOperatorsRule());
prepareForJobGenRewrites
.add(new IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
+ prepareForJobGenRewrites.add(new FixReplicateOperatorOutputsRule());
prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
// Re-infer all types, so that, e.g., the effect of not-is-null is
// propagated.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
new file mode 100644
index 0000000..7b13a2b
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFullParallelSortRule.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * <pre>
+ * Description:
+ * This rule checks whether full parallel sort is applicable to {@link OrderOperator}. It disables full parallel
+ * sort when a limit operator or running aggregate operator is present in the parents of the order operator.
+ * Pre-conditions:
+ * None.
+ * Post-requirements:
+ * 1. {@link org.apache.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule}
+ * </pre>
+ */
+public class CheckFullParallelSortRule implements IAlgebraicRewriteRule {
+ private final List<AbstractLogicalOperator> parents = new ArrayList<>();
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ parents.add((AbstractLogicalOperator) opRef.getValue());
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx) throws AlgebricksException {
+ parents.remove(parents.size() - 1);
+ AbstractLogicalOperator orderOp = (AbstractLogicalOperator) opRef.getValue();
+ if (orderOp.getOperatorTag() == LogicalOperatorTag.ORDER
+ && !orderOp.getAnnotations().containsKey(OperatorAnnotations.USE_STATIC_RANGE)) {
+ // disable when sort output is consumed by limit & running agg op (result should be unpartitioned for limit)
+ AbstractLogicalOperator parent;
+ for (int i = parents.size() - 1; i >= 0; i--) {
+ parent = parents.get(i);
+ if (parent.getOperatorTag() == LogicalOperatorTag.LIMIT
+ && ((LimitOperator) parent).isTopmostLimitOp()) {
+ orderOp.getAnnotations().put(OperatorAnnotations.USE_DYNAMIC_RANGE, Boolean.FALSE);
+ return true;
+ }
+ if (parent.getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) {
+ orderOp.getAnnotations().put(OperatorAnnotations.USE_DYNAMIC_RANGE, Boolean.FALSE);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
index bd79a38..dc135c9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FixReplicateOperatorOutputsRule.java
@@ -140,7 +140,7 @@ public class FixReplicateOperatorOutputsRule implements IAlgebraicRewriteRule {
boolean parentFixed = false;
for (int oldParentIndex = 0; oldParentIndex < replicateOperator.getOutputs().size(); oldParentIndex++) {
if (parentsPathToReplicate.contains(replicateOperator.getOutputs().get(oldParentIndex))) {
- replicateOperator.getOutputs().get(oldParentIndex).setValue(replicateActualParent.getValue());
+ replicateOperator.getOutputs().set(oldParentIndex, replicateActualParent);
parentFixed = true;
updateNumberOfParentsDone(replicateOperator);
break;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 8372851..0c91e9b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -34,11 +34,12 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCa
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -101,12 +102,10 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule
private class IllegalNonfunctionalFunctionSweeperOperatorVisitor implements ILogicalOperatorVisitor<Void, Void> {
private void sweepExpression(ILogicalExpression expr, ILogicalOperator op) throws AlgebricksException {
- if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- if (!expr.isFunctional()) {
- AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, fce.getSourceLocation(),
- "Found non-functional function " + fce.getFunctionIdentifier() + " in op " + op);
- }
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL && !expr.isFunctional()) {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, fce.getSourceLocation(),
+ "Found non-functional function " + fce.getFunctionIdentifier() + " in op " + op);
}
}
@@ -308,6 +307,12 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule
public Void visitTokenizeOperator(TokenizeOperator op, Void tag) throws AlgebricksException {
return null;
}
+
+ @Override
+ public Void visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ sweepExpression(op.getRangeMapExpression().getValue(), op);
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 4d646f3..22ef303 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -59,6 +59,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -642,6 +643,12 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
return visitSingleInputOperator(op);
}
+ @Override
+ public ILogicalOperator visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, op.getSourceLocation(),
+ "Forward operator should have been disqualified for this rewriting!");
+ }
+
/**
* Wraps an AggregateOperator or RunningAggregateOperator with a group-by
* operator where the group-by keys are variables in keyVarsToEnforce. Note
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 610fea2..b862a6f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -42,6 +42,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -52,6 +53,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperato
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
@@ -63,23 +65,19 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.visitors.IQueryOperatorVisitor;
-/*
- This visitor inlines the input <code>nts</code> in the query plan rooted
- at the operator being visited, with the query plan rooted at the input
- <code>subplanInputOperator</code>.
-
- The visitor ensures that:
- 1. live variables at <code>subplanInputOperator</code> are
- propagated to the top-most join operator in the query plan rooted
- at the operator being visited;
- 2. no available tuple at <code>subplanInputOperator</code> get lost along the
- pipeline to the top-most join operator in the query plan rooted
- at the operator being visited.
-*/
+/**
+ * This visitor inlines the input {@code nts} in the query plan rooted at the operator being visited,
+ * with the query plan rooted at the input {@code subplanInputOperator}.
+ *
+ * The visitor ensures that:
+ * 1. live variables at {@code subplanInputOperator} are propagated to the top-most join operator in the query plan
+ * rooted at the operator being visited.
+ * 2. no available tuple at {@code subplanInputOperator} get lost along the pipeline to the top-most join operator
+ * in the query plan rooted at the operator being visited.
+ */
class InlineLeftNtsInSubplanJoinFlatteningVisitor implements IQueryOperatorVisitor<ILogicalOperator, Void> {
// The optimization context.
private final IOptimizationContext context;
@@ -380,6 +378,12 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor implements IQueryOperatorVisit
return visitSingleInputOperator(op);
}
+ @Override
+ public ILogicalOperator visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ throw new UnsupportedOperationException(
+ "Nested subplans with a forward operator should have been disqualified for this rewriting!");
+ }
+
private ILogicalOperator visitSingleInputOperator(ILogicalOperator op) throws AlgebricksException {
if (op.getInputs().size() == 1) {
// Deals with single input operators.
@@ -395,8 +399,7 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor implements IQueryOperatorVisit
}
/**
- * Inject varaibles to indicate non-matches for the right branch of
- * a left-outer join.
+ * Inject variables to indicate non-matches for the right branch of a left-outer join.
*
* @param joinOp
* the leftouter join operator.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index 44bfbe4..e2b104d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -18,16 +18,19 @@
*/
package org.apache.asterix.optimizer.rules.subplan;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -229,6 +232,12 @@ class SubplanSpecialFlatteningCheckVisitor implements IQueryOperatorVisitor<Bool
return visitInputs(op);
}
+ @Override
+ public Boolean visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, op.getSourceLocation(),
+ "Forward operator should have been disqualified for this rewriting!");
+ }
+
private boolean visitInputs(ILogicalOperator op) throws AlgebricksException {
for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
if (childRef.getValue().accept(this, null)) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 91de474..19fe02d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -38,7 +38,6 @@ import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.lang.aql.util.RangeMapBuilder;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
@@ -69,6 +68,7 @@ import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.common.util.RangeMapBuilder;
import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.DataSource;
@@ -1163,9 +1163,9 @@ class LangExpressionToPlanTranslator
}
if (oc.getRangeMap() != null) {
Iterator<OrderModifier> orderModifIter = oc.getModifierList().iterator();
- boolean ascending = (orderModifIter.next() == OrderModifier.ASC);
+ boolean ascending = orderModifIter.next() == OrderModifier.ASC;
RangeMapBuilder.verifyRangeOrder(oc.getRangeMap(), ascending);
- ord.getAnnotations().put(OperatorAnnotations.USE_RANGE_CONNECTOR, oc.getRangeMap());
+ ord.getAnnotations().put(OperatorAnnotations.USE_STATIC_RANGE, oc.getRangeMap());
}
return new Pair<>(ord, null);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index cfec2de..9727d46 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -36,7 +36,6 @@ import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
import org.apache.asterix.api.http.server.ResultUtil;
import org.apache.asterix.common.api.INodeJobTracker;
import org.apache.asterix.common.config.CompilerProperties;
-import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -136,7 +135,8 @@ public class APIFramework {
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
- CompilerProperties.COMPILER_PARALLELISM_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ CompilerProperties.COMPILER_PARALLELISM_KEY, CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
+ CompilerProperties.COMPILER_SORT_SAMPLES_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
@@ -336,12 +336,17 @@ public class APIFramework {
int textSearchFrameLimit = getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
(String) querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY),
compilerProperties.getTextSearchMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_TEXTSEARCH, sourceLoc);
- final PhysicalOptimizationConfig physOptConf = OptimizationConfUtil.getPhysicalOptimizationConfig();
+ int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig);
+ boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig);
+
+ final PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
physOptConf.setMaxFramesExternalGroupBy(groupFrameLimit);
physOptConf.setMaxFramesForJoin(joinFrameLimit);
physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
+ physOptConf.setSortParallel(fullParallelSort);
+ physOptConf.setSortSamples(sortNumSamples);
return physOptConf;
}
@@ -495,6 +500,25 @@ public class APIFramework {
return parameter == null ? parallelismInConfiguration : integerIPropertyInterpreter.parse(parameter);
}
+ private boolean getSortParallel(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig) {
+ String valueInQuery = (String) querySpecificConfig.get(CompilerProperties.COMPILER_SORT_PARALLEL_KEY);
+ if (valueInQuery != null) {
+ return OptionTypes.BOOLEAN.parse(valueInQuery);
+ }
+ return compilerProperties.getSortParallel();
+ }
+
+ private int getSortSamples(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig) {
+ String valueInQuery = (String) querySpecificConfig.get(CompilerProperties.COMPILER_SORT_SAMPLES_KEY);
+ if (valueInQuery != null) {
+ int parsedNumSamples = OptionTypes.INTEGER.parse(valueInQuery);
+ if (parsedNumSamples > 0) {
+ return parsedNumSamples;
+ }
+ }
+ return compilerProperties.getSortSamples();
+ }
+
// Validates if the query contains unsupported query parameters.
private static Map<String, Object> validateConfig(Map<String, Object> config, SourceLocation sourceLoc)
throws AlgebricksException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
index 1913683..c7db521 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/OperatorResourcesComputer.java
@@ -86,6 +86,7 @@ public class OperatorResourcesComputer {
case INDEX_INSERT_DELETE_UPSERT:
case INSERT_DELETE_UPSERT:
case INTERSECT:
+ case FORWARD:
return getOperatorRequiredMemory(operator, frameSize);
case LEFT_OUTER_UNNEST_MAP:
case UNNEST_MAP:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
index 8b32375..0023a7a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/PlanStagesGenerator.java
@@ -38,6 +38,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -67,13 +68,22 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteResultO
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
import org.apache.hyracks.util.annotations.NotThreadSafe;
+/**
+ * Visits the operator first. Then, it visits all its inputs (pre-order traversal). When it visits an operator, it adds
+ * the operator to the current stage. If the operator is a multi-stage operator, it also adds the operator to a queue
+ * to re-visit the operator again to create the other stage.
+ */
@NotThreadSafe
public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, Void> {
- private static final int JOIN_FIRST_INPUT = 1;
- private static final int JOIN_SECOND_INPUT = 2;
+ private static final int JOIN_NON_BLOCKING_INPUT = 0;
+ private static final int JOIN_BLOCKING_INPUT = 1;
+ private static final int JOIN_NUM_INPUTS = 2;
+ private static final int FORWARD_NON_BLOCKING_INPUT = 0;
+ private static final int FORWARD_BLOCKING_INPUT = 1;
+ private static final int FORWARD_NUM_INPUTS = 2;
private final Set<ILogicalOperator> visitedOperators = new HashSet<>();
- private final LinkedList<ILogicalOperator> pendingBlockingOperators = new LinkedList<>();
+ private final LinkedList<ILogicalOperator> pendingMultiStageOperators = new LinkedList<>();
private final List<PlanStage> stages = new ArrayList<>();
private PlanStage currentStage;
private int stageCounter;
@@ -163,7 +173,7 @@ public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, Void>
@Override
public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
- // Makes sure that the downstream of a replicate operator is only visited once.
+ // make sure that the downstream of a replicate operator is visited only once.
if (!visitedOperators.contains(op)) {
visitedOperators.add(op);
visit(op);
@@ -175,7 +185,7 @@ public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, Void>
@Override
public Void visitSplitOperator(SplitOperator op, Void arg) throws AlgebricksException {
- // Makes sure that the downstream of a split operator is only visited once.
+ // make sure that the downstream of a split operator is visited only once.
if (!visitedOperators.contains(op)) {
visitedOperators.add(op);
visit(op);
@@ -300,59 +310,82 @@ public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, Void>
return null;
}
+ @Override
+ public Void visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ visit(op);
+ return null;
+ }
+
public List<PlanStage> getStages() {
return stages;
}
private void visit(ILogicalOperator op) throws AlgebricksException {
addToStage(op);
- if (!pendingBlockingOperators.isEmpty()) {
- final ILogicalOperator firstPending = pendingBlockingOperators.pop();
- visitBlocking(firstPending);
+ if (!pendingMultiStageOperators.isEmpty()) {
+ final ILogicalOperator firstPending = pendingMultiStageOperators.pop();
+ visitMultiStageOp(firstPending);
}
}
- private void visitBlocking(ILogicalOperator blockingOp) throws AlgebricksException {
+ private void visitMultiStageOp(ILogicalOperator multiStageOp) throws AlgebricksException {
final PlanStage blockingOpStage = new PlanStage(++stageCounter);
- blockingOpStage.getOperators().add(blockingOp);
+ blockingOpStage.getOperators().add(multiStageOp);
stages.add(blockingOpStage);
currentStage = blockingOpStage;
- switch (blockingOp.getOperatorTag()) {
+ switch (multiStageOp.getOperatorTag()) {
case INNERJOIN:
case LEFTOUTERJOIN:
- // visit only the second input
- ILogicalOperator joinSecondInput = getJoinOperatorInput(blockingOp, JOIN_SECOND_INPUT);
- joinSecondInput.accept(this, null);
+ // visit only the blocking input creating a new stage
+ ILogicalOperator newStageOperator = getInputAt(multiStageOp, JOIN_BLOCKING_INPUT, JOIN_NUM_INPUTS);
+ newStageOperator.accept(this, null);
break;
case GROUP:
case ORDER:
- visitInputs(blockingOp);
+ visitInputs(multiStageOp);
+ break;
+ case FORWARD:
+ // visit only the blocking input creating a new stage
+ ILogicalOperator newStageOp = getInputAt(multiStageOp, FORWARD_BLOCKING_INPUT, FORWARD_NUM_INPUTS);
+ newStageOp.accept(this, null);
break;
default:
- throw new IllegalStateException("Unrecognized blocking operator: " + blockingOp.getOperatorTag());
+ throw new IllegalStateException("Unrecognized blocking operator: " + multiStageOp.getOperatorTag());
}
}
+ /**
+ * Adds the op argument to the current stage. If the operator is a multi-stage, it adds the operator to the pending
+ * list and continues on the branch that is non-blocking (i.e., the branch continuing on the same current stage)
+ * @param op to be added to the current stage
+ * @throws AlgebricksException
+ */
private void addToStage(ILogicalOperator op) throws AlgebricksException {
currentStage.getOperators().add(op);
switch (op.getOperatorTag()) {
case INNERJOIN:
case LEFTOUTERJOIN:
- pendingBlockingOperators.add(op);
+ pendingMultiStageOperators.add(op);
// continue on the same stage
- final ILogicalOperator joinFirstInput = getJoinOperatorInput(op, JOIN_FIRST_INPUT);
- joinFirstInput.accept(this, null);
+ final ILogicalOperator joinNonBlockingInput = getInputAt(op, JOIN_NON_BLOCKING_INPUT, JOIN_NUM_INPUTS);
+ joinNonBlockingInput.accept(this, null);
break;
case GROUP:
if (isBlockingGroupBy((GroupByOperator) op)) {
- pendingBlockingOperators.add(op);
+ pendingMultiStageOperators.add(op);
return;
}
// continue on the same stage
visitInputs(op);
break;
case ORDER:
- pendingBlockingOperators.add(op);
+ pendingMultiStageOperators.add(op);
+ break;
+ case FORWARD:
+ pendingMultiStageOperators.add(op);
+ // continue on the same current stage through the branch that is non-blocking
+ ILogicalOperator nonBlockingInput = getInputAt(op, FORWARD_NON_BLOCKING_INPUT, FORWARD_NUM_INPUTS);
+ nonBlockingInput.accept(this, null);
break;
default:
visitInputs(op);
@@ -397,15 +430,16 @@ public class PlanStagesGenerator implements ILogicalOperatorVisitor<Void, Void>
return false;
}
- private ILogicalOperator getJoinOperatorInput(ILogicalOperator op, int inputNum) {
- if (inputNum != JOIN_FIRST_INPUT && inputNum != JOIN_SECOND_INPUT) {
- throw new IllegalArgumentException("invalid input number for join operator");
- }
+ private ILogicalOperator getInputAt(ILogicalOperator op, int inputIndex, int numInputs) {
final List<Mutable<ILogicalOperator>> inputs = op.getInputs();
- if (inputs.size() != 2) {
- throw new IllegalStateException("Join must have exactly two inputs. Current inputs: " + inputs.size());
+ int inSize = inputs.size();
+ if (inSize != numInputs) {
+ throw new IllegalStateException("Op must have exactly " + numInputs + " inputs. Current inputs: " + inSize);
+ }
+ if (inputIndex >= inSize) {
+ throw new IllegalArgumentException("invalid input index for operator");
}
- return op.getInputs().get(inputNum - 1).getValue();
+ return inputs.get(inputIndex).getValue();
}
/**
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
index 89fbcb2..368a244 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/resource/RequiredCapacityVisitor.java
@@ -36,6 +36,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ForwardOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -304,6 +305,12 @@ public class RequiredCapacityVisitor implements ILogicalOperatorVisitor<Void, Vo
return null;
}
+ @Override
+ public Void visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
+ visitInternal(op, true);
+ return null;
+ }
+
// Calculates the memory usage for exchange operators.
private void calculateMemoryUsageForExchange(ExchangeOperator op) throws AlgebricksException {
visitInternal(op, false);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index 5f76568..d0adcda 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -23,7 +23,6 @@ import java.io.DataOutputStream;
import java.util.HashMap;
import java.util.Map;
-import org.apache.asterix.external.library.java.base.ByteArrayAccessibleInputStream;
import org.apache.asterix.external.parser.JSONDataParser;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.om.base.IAObject;
@@ -35,6 +34,7 @@ import org.apache.asterix.translator.ResultProperties;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
import com.fasterxml.jackson.databind.JsonNode;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/main/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 6971b2b..8877be8 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -53,6 +53,7 @@ compiler.sortmemory=320KB
compiler.groupmemory=160KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
+compiler.sort.parallel=false
messaging.frame.size=4096
messaging.frame.count=512
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
index 385f6a2..66b01a7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPartitionComputerFactory.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputer;
import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -41,7 +42,7 @@ public class TestPartitionComputerFactory implements ITuplePartitionComputerFact
}
@Override
- public ITuplePartitionComputer createPartitioner() {
+ public ITuplePartitionComputer createPartitioner(IHyracksTaskContext ctx) {
return new ITuplePartitionComputer() {
private final List<Integer> destinations =
new ArrayList<Integer>(TestPartitionComputerFactory.this.destinations);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index debb3f9..6e0413c 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -60,6 +60,11 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+/**
+ * Runs the optimization tests. The current configuration runs the tests with parallel sort disabled.
+ * Note: when adding a new test case and it includes sorting, provide another test case and enable parallel sort in the
+ * query by setting the property (append the test case name with "_ps")
+ */
@RunWith(Parameterized.class)
public class OptimizerTest {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
new file mode 100644
index 0000000..4d8c4ff
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01_ps.sqlpp
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ `screen-name` : string,
+ lang : string,
+ `friends-count` : integer,
+ `statuses-count` : integer,
+ name : string,
+ `followers-count` : integer
+};
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : bigint,
+ user : TwitterUserType,
+ `sender-location` : point,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string,
+ countA : integer,
+ countB : integer
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages (`sender-location`) type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgTextIx on TweetMessages (`message-text`) type keyword;
+
+write output to asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_01.adm";
+set `compiler.sort.parallel` "true";
+
+select element {'tweetid1':t1.tweetid,'count1':t1.countA,'t2info':(
+ select element {'tweetid2':t2.tweetid,'count2':t2.countB}
+ from TweetMessages as t2
+ where (t1.countA /*+ indexnl */ = t2.countB)
+ order by t2.tweetid
+ )}
+from TweetMessages as t1
+where (t1.tweetid < test.bigint('10'))
+order by t1.tweetid
+;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
new file mode 100644
index 0000000..a5c5fe4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_02_ps.sqlpp
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and
+ * another for secondary btree index in index subtree. p_sort enabled.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ `screen-name` : string,
+ lang : string,
+ `friends-count` : integer,
+ `statuses-count` : integer,
+ name : string,
+ `followers-count` : integer
+};
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : bigint,
+ user : TwitterUserType,
+ `sender-location` : point,
+ `send-time` : datetime,
+ `referred-topics` : {{string}},
+ `message-text` : string,
+ countA : integer,
+ countB : integer
+};
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages (`sender-location`) type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgTextIx on TweetMessages (`message-text`) type keyword;
+
+write output to asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_02.adm";
+
+set `compiler.sort.parallel` "true";
+
+select element {'tweetid1':t1.tweetid,'count1':t1.countA,'t2info':(
+ select element {'tweetid2':t2.tweetid,'count2':t2.countB}
+ from TweetMessages as t2
+ where ((t1.countA /*+ indexnl */ = t2.countB) and (t1.tweetid != t2.tweetid))
+ order by t2.tweetid
+ )}
+from TweetMessages as t1
+where (t1.tweetid < test.bigint('10'))
+order by t1.tweetid
+;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
new file mode 100644
index 0000000..520a3f0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-68_ps.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index intersection with between operator (ASTERIXDB-2448). p_sort enabled.
+ * Expected Result : Success
+ */
+
+drop dataverse tpch if exists;
+create dataverse tpch;
+
+use tpch;
+
+create type tpch.OrderType as
+ closed {
+ o_orderkey : bigint,
+ o_custkey : bigint,
+ o_orderstatus : string,
+ o_totalprice : double,
+ o_orderdate : string,
+ o_orderpriority : string,
+ o_clerk : string,
+ o_shippriority : bigint,
+ o_comment : string
+};
+
+create dataset Orders(OrderType) primary key o_orderkey;
+
+create index idx_custkey on Orders (o_custkey) type btree;
+
+create index idx_orderpriority on Orders (o_orderpriority) type btree;
+
+set `compiler.sort.parallel` "true";
+
+select o_custkey, o_orderkey, o_orderstatus from Orders
+where
+ o_orderpriority = '1-URGENT' and
+ o_custkey between 40 and 43
+order by o_custkey, o_orderkey;
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
new file mode 100644
index 0000000..b986780
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01-disable-idxonly_ps.sqlpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan verification test
+ * : The test is intended to verify that the secondary BTree index is used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split -> -> union -> ...
+ * : However, we set the "noindexonly" option to true. So, the index-only plan should not be triggered.
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+set noindexonly 'true';
+set `compiler.sort.parallel` "true";
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
new file mode 100644
index 0000000..e5c4e0c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-sidx-idxonly-01_ps.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan verification test
+ * : The test is intended to verify that the secondary BTree index is used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split -> -> union -> ...
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+set `compiler.sort.parallel` "true";
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/80225e2c/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
index 103aae8..9921e58 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/introhashpartitionmerge.sqlpp
@@ -36,6 +36,9 @@ create nodegroup group1 if not exists on
create dataset TOKENSRANKEDADM(TOKENSRANKEDADMType) primary key rank on group1;
write output to asterix_nc1:"rttest/introhashpartitionmerge.adm";
+
+SET `compiler.sort.parallel` "false";
+
select element token2.rank
from TOKENSRANKEDADM as token1,
(
@@ -44,5 +47,4 @@ from TOKENSRANKEDADM as token1,
order by tokenRanked.rank
) as token2
where (token1.token = token2.token)
-order by token2.rank
-;
+order by token2.rank;