You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2021/04/07 12:47:41 UTC
[asterixdb] branch master updated (b0be3fc -> 21cf9cb)
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from b0be3fc [NO ISSUE][COMP] Avoid variable reference sharing
new 690da33 [NO ISSUE][OTH] Update storage options names
new 6c5903a [NO ISSUE][COMP] Improve subplan inlining
new 2e9815a [NO ISSUE][COMP] Change default for synonym's target object dataverse
new a016c11 [ASTERIXDB-2854] Make UDF deploy timeout configurable
new 831b981 [NO ISSUE][*DB] Update Dataverse canonical form
new 1dea458 [ASTERIXDB-2855] Allow additions to Python UDF env
new 2779208 [NO ISSUE][EXT] Avoid duplicate open for streams + minor refactoring
new e48e9f2 [ASTERIXDB-2857][RT] Incorrect result for nested loop outer join
new d382766 [NO ISSUE][EXT]: Improve workload distribution logic
new 17670aa [NO ISSUE][STO] Delete invalid indexes during cluster global recovery
new 8555239 Revert "[NO ISSUE][API] Decode request local path"
new 9c01346 [NO ISSUE][OTH] Use IntOpenHashSet for datasets ids
new 778db56 [NO ISSUE][COMP] Eliminate unused running aggregates
new 7b2191a [NO ISSUE][OTH] Add an identifier mapper to map identifiers
new c3dbe4b [NO ISSUE][REP] Log details of files to be replicated
new 104d46a [NO ISSUE][*DB][TXN] Avoid abort log on empty txn, reduce recovery logging
new d4f3aba [NO ISSUE][*DB][MISC] Avoid appending trailing ; on statements ending with a ;
new d18d6ac [NO ISSUE][*DB][ACT] += ActiveManagerMessage.GENERIC_EVENT
new d821af0 [NO ISSUE][MISC] Advance it.unimi.dsi:fastutil 8.3.0->8.5.4
new 244c3c5 [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry delay
new 5b5ac48 [ASTERIXDB-2858][EXT]: Retry upon failure for S3 retryable errors
new c74bdc8 [NO ISSUE][EXT] Make read buffer size configurable
new 3ea8ef1 [ASTERIXDB-2870][EXT]: Close client after it is used
new ce5f3aa Merge mad-hatter into cheshire-cat
new 21cf9cb Merge branch 'cheshire-cat' into 'master'
The 25 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/asterix/active/ActiveManager.java | 21 +-
.../org/apache/asterix/active/IActiveRuntime.java | 5 +
.../active/message/ActiveManagerMessage.java | 14 +-
.../active/message/ActiveStatsRequestMessage.java | 6 +-
.../active/message/StopRuntimeParameters.java | 12 +-
.../operators/physical/InvertedIndexPOperator.java | 4 +-
.../optimizer/rules/ConstantFoldingRule.java | 3 +-
.../rules/MetaFunctionToMetaVariableRule.java | 4 +-
.../rules/SetAsterixPhysicalOperatorsRule.java | 5 +-
.../rules/SweepIllegalNonfunctionalFunctions.java | 3 -
.../subplan/InlineAllNtsInSubplanVisitor.java | 84 +-
...InlineSubplanInputForNestedTupleSourceRule.java | 48 +-
.../rules/subplan/SubplanFlatteningUtil.java | 51 +-
.../rules/util/EquivalenceClassUtils.java | 84 +-
.../asterix/translator/AbstractLangTranslator.java | 21 +-
.../translator/LangExpressionToPlanTranslator.java | 22 +-
.../apache/asterix/api/common/APIFramework.java | 7 +-
.../api/http/server/ConnectorApiServlet.java | 7 +-
.../asterix/api/http/server/NCUdfApiServlet.java | 4 +-
.../api/http/server/QueryServiceServlet.java | 14 +-
.../api/http/server/RebalanceApiServlet.java | 5 +-
.../app/active/ActiveEntityEventsListener.java | 2 +-
.../asterix/app/function/DatasetRewriter.java | 6 +-
.../apache/asterix/app/function/FeedRewriter.java | 5 +-
.../app/message/StorageCleanupRequestMessage.java | 109 +
.../apache/asterix/app/message/VoidResponse.java | 63 +
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +-
.../org/apache/asterix/app/nc/RecoveryManager.java | 19 +-
.../asterix/app/translator/QueryTranslator.java | 43 +-
.../hyracks/bootstrap/GlobalRecoveryManager.java | 31 +
.../test/dataflow/GlobalStorageCleanupTest.java | 68 +
.../test/dataflow/LSMFlushRecoveryTest.java | 12 +-
.../src/test/resources/TweetSent/roundtrip.py | 4 +
asterixdb/asterix-app/src/test/resources/cc.conf | 1 +
.../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 +
.../queries/subquery/in_let_8.sqlpp} | 43 +-
.../queries/subquery/query-ASTERIXDB-2845.sqlpp | 2 +-
.../optimizerts/results/ASTERIXDB-2402.plan | 28 +-
.../leftouterjoin/query-ASTERIXDB-2857.plan | 39 +
.../optimizerts/results/query-issue562_ps.plan | 58 +-
.../optimizerts/results/subquery/exists.plan | 2 +-
.../optimizerts/results/subquery/exists_ps.plan | 4 +-
.../optimizerts/results/subquery/in_let_3.plan | 4 +-
.../optimizerts/results/subquery/in_let_4.plan | 4 +-
.../optimizerts/results/subquery/in_let_5.plan | 4 +-
.../subquery/{in_let_3.plan => in_let_8.plan} | 60 +-
.../optimizerts/results/subquery/not_exists.plan | 2 +-
.../results/subquery/not_exists_ps.plan | 4 +-
.../results/subquery/query-ASTERIXDB-2845.plan | 145 +-
.../tpcds/query-ASTERIXDB-1581-correlated-2.plan | 399 +-
.../tpcds/query-ASTERIXDB-1581-correlated.plan | 407 +-
.../external_dataset.000.ddl.sqlpp | 66 +
.../external_dataset.001.query.sqlpp} | 7 +-
.../external_dataset.002.query.sqlpp} | 7 +-
.../external_dataset.003.query.sqlpp} | 7 +-
.../external_dataset.004.query.sqlpp} | 7 +-
.../external_dataset.005.query.sqlpp} | 7 +-
.../external_dataset.006.query.sqlpp} | 7 +-
.../external_dataset.999.ddl.sqlpp} | 0
.../py_function_error.2.ddl.sqlpp | 5 +-
...2.ddl.sqlpp => py_function_error.5.query.sqlpp} | 12 +-
...2.ddl.sqlpp => py_function_error.6.query.sqlpp} | 12 +-
.../query-ASTERIXDB-2857.1.ddl.sqlpp} | 30 +-
.../query-ASTERIXDB-2857.2.update.sqlpp} | 0
.../query-ASTERIXDB-2857.3.query.sqlpp} | 23 +-
.../in_let/in_let.8.query.sqlpp} | 21 +-
.../synonym-03/synonym-03.1.ddl.sqlpp} | 19 +-
.../synonym-03/synonym-03.2.query.sqlpp} | 9 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +
.../cluster_state_1_full.1.regexadm | 2 +
.../cluster_state_1_less.1.regexadm | 2 +
.../results/binary/query_id/query_id.1.adm | 6005 --------------------
.../external_dataset.001.adm | 0
.../external_dataset.002.adm | 0
.../external_dataset.003.adm | 0
.../external_dataset.004.adm | 0
.../external_dataset.005.adm | 0
.../external_dataset.006.adm | 0
.../py_function_error/py_function_error.3.json | 1 +
.../py_function_error/py_function_error.4.json | 1 +
.../query-ASTERIXDB-2857.3.adm | 2 +
.../special_chars_2/special_chars_2.2.adm | 6 +-
.../subquery/in_let/{in_let.7.adm => in_let.8.adm} | 0
.../results/synonym/synonym-03/synonym-03.2.adm | 2 +
.../resources/runtimets/testsuite_it_python.xml | 2 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 41 +-
.../asterix/common/api/IIdentifierMapper.java} | 11 +-
.../asterix/common/config/CompilerProperties.java | 17 +-
.../asterix/common/config/ExternalProperties.java | 8 +-
.../common/config/OptimizationConfUtil.java | 15 +
.../asterix/common/config/StorageProperties.java | 54 +-
.../common/config/TransactionProperties.java | 5 +-
.../asterix/common/exceptions/WarningUtil.java | 4 +-
.../asterix/common/metadata/DataverseName.java | 74 +-
.../common/utils/IdentifierMappingUtil.java} | 25 +-
.../asterix/common/utils/IdentifierUtil.java} | 18 +-
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../asterix/common/metadata/DataverseNameTest.java | 25 +-
.../asterix-doc/src/main/markdown/sqlpp/5_error.md | 2 +-
.../src/main/markdown/sqlpp/7_ddl_dml.md | 3 +-
.../apache/asterix/external/api/IRecordReader.java | 5 +
.../dataflow/AbstractFeedDataFlowController.java | 5 +
.../dataflow/FeedRecordDataFlowController.java | 6 +
.../external/dataset/adapter/FeedAdapter.java | 5 +
.../abstracts/AbstractExternalInputStream.java | 3 -
.../AbstractExternalInputStreamFactory.java | 32 +-
.../input/record/reader/aws/AwsS3InputStream.java | 82 +-
.../record/reader/aws/AwsS3InputStreamFactory.java | 30 +-
.../record/reader/azure/AzureBlobInputStream.java | 12 +-
.../reader/azure/AzureBlobInputStreamFactory.java | 30 +-
.../record/reader/stream/StreamRecordReader.java | 6 +-
.../input/stream/AsterixInputStreamReader.java | 11 +-
.../StandardUTF8ToModifiedUTF8DataOutput.java | 2 +-
.../external/library/PythonLibraryEvaluator.java | 17 +-
.../library/PythonLibraryEvaluatorFactory.java | 34 +-
.../external/util/ExternalDataConstants.java | 14 +-
.../asterix/external/util/ExternalDataUtils.java | 13 +-
.../input/record/reader/awss3/AwsS3Test.java | 145 +
.../org/apache/asterix/metadata/MetadataNode.java | 14 +-
.../metadata/declared/DatasetDataSource.java | 6 +
.../metadata/declared/MetadataManagerUtil.java | 8 +-
.../metadata/declared/MetadataProvider.java | 17 +-
.../apache/asterix/metadata/entities/Function.java | 7 +
.../asterix/metadata/feeds/FeedMetadataUtil.java | 9 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 6 +-
.../InvertedIndexResourceFactoryProvider.java | 5 +-
.../asterix/om/functions/BuiltinFunctions.java | 2 +-
.../replication/management/NetworkingUtil.java | 16 +-
.../replication/messaging/ReplicateFileTask.java | 7 +
.../asterix/replication/sync/FileSynchronizer.java | 4 +
.../functions/CreateQueryUIDDescriptor.java | 32 +-
.../service/transaction/TransactionManager.java | 12 +-
asterixdb/pom.xml | 2 +-
.../appended-resources/supplemental-models.xml | 32 +-
.../common/exceptions/AlgebricksException.java | 2 +
.../api/HeuristicCompilerFactoryBuilder.java | 3 +-
.../algebricks/algebricks-core/pom.xml | 2 +-
.../logical/RunningAggregateOperator.java | 5 +
.../algebra/util/OperatorManipulationUtil.java | 9 +-
.../algebricks/core/config/AlgebricksConfig.java | 3 +
.../algebricks/core/jobgen/impl/JobGenContext.java | 9 +-
.../rewriter/base/PhysicalOptimizationConfig.java | 9 +
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 +-
.../hyracks/api/exceptions/HyracksException.java | 2 +
.../api/exceptions/IFormattedException.java | 11 +
.../org/apache/hyracks/api/exceptions/Warning.java | 27 +-
.../org/apache/hyracks/api/util/InvokeUtil.java | 2 +-
.../control/common/controllers/NCConfig.java | 4 +
.../hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +-
.../hyracks/dataflow/common/io/RunFileWriter.java | 10 +-
.../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 +-
.../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +-
.../hyracks/http/server/AbstractServlet.java | 9 +-
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 2 +-
hyracks-fullstack/pom.xml | 4 +-
155 files changed, 2368 insertions(+), 7210 deletions(-)
create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java
create mode 100644 asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java
create mode 100644 asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
rename asterixdb/asterix-app/src/test/resources/{runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp => optimizerts/queries/subquery/in_let_8.sqlpp} (60%)
create mode 100644 asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
copy asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/{in_let_3.plan => in_let_8.plan} (67%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp} (88%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.2.update.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp} (83%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{warnings/warnings-limit/warnings-limit.09.ddl.sqlpp => external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/{py_function_error.2.ddl.sqlpp => py_function_error.5.query.sqlpp} (82%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/{py_function_error.2.ddl.sqlpp => py_function_error.6.query.sqlpp} (82%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/py_function_error/py_function_error.2.ddl.sqlpp => leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp} (55%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{rightouterjoin/roj-01-core/roj-01-core.2.update.sqlpp => leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/py_function_error/py_function_error.2.ddl.sqlpp => leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp} (62%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/py_function_error/py_function_error.2.ddl.sqlpp => subquery/in_let/in_let.8.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/py_function_error/py_function_error.2.ddl.sqlpp => synonym/synonym-03/synonym-03.1.ddl.sqlpp} (70%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{binary/query_id/query_id.3.query.sqlpp => synonym/synonym-03/synonym-03.2.query.sqlpp} (85%)
delete mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.001.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.002.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.003.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.004.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.005.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/{json/json => custom-buffer-size}/external_dataset.006.adm (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
copy asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/{in_let.7.adm => in_let.8.adm} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm
rename asterixdb/{asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp => asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java} (85%)
copy asterixdb/{asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp => asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java} (60%)
copy asterixdb/{asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp => asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java} (68%)
create mode 100644 asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
[asterixdb] 20/25: [ASTERIXDB-2870][EXT]: Close client after it is
used
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3ea8ef15a2aa47fd3d6e87cb19acd5254efa9252
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Tue Apr 6 04:00:34 2021 +0300
[ASTERIXDB-2870][EXT]: Close client after it is used
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Close the S3 client after it is used to release
the connections.
Change-Id: I8611b5a05fcbd8a4a9a4556c290281fd5cbd56a4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10885
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann
---
.../input/record/reader/aws/AwsS3InputStream.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index f558820..8bd7a51 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -30,6 +30,7 @@ import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.stream.AbstractMultipleInputStream;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.util.LogRedactionUtil;
@@ -46,7 +47,7 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
private static final Logger LOGGER = LogManager.getLogger();
// Configuration
- private final Map<String, String> configuration;
+ private final String bucket;
private final int bufferSize;
private final S3Client s3Client;
@@ -56,10 +57,10 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
private int nextFileIndex = 0;
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
- this.configuration = configuration;
this.filePaths = filePaths;
this.s3Client = buildAwsS3Client(configuration);
this.bufferSize = ExternalDataUtils.getOrDefaultBufferSize(configuration);
+ this.bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
}
@Override
@@ -71,9 +72,6 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
// Finished reading all the files
if (nextFileIndex >= filePaths.size()) {
- if (in != null) {
- CleanupUtils.close(in, null);
- }
return false;
}
@@ -82,9 +80,9 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
CleanupUtils.close(in, null);
}
- String bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
- GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(filePaths.get(nextFileIndex)).build();
+ GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(fileName).build();
// Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
// the header, then the S3 stream gets closed in the close method
@@ -100,9 +98,8 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
}
// Use gzip stream if needed
- String filename = filePaths.get(nextFileIndex).toLowerCase();
- if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(s3Client.getObject(getObjectRequest), bufferSize);
+ if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
+ in = new GZIPInputStream(in, bufferSize);
}
// Current file ready, point to the next file
@@ -136,6 +133,9 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
if (in != null) {
CleanupUtils.close(in, null);
}
+ if (s3Client != null) {
+ CleanupUtils.close(s3Client, null);
+ }
}
@Override
[asterixdb] 06/25: [NO ISSUE][*DB] Update Dataverse canonical form
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 831b981dc5b0a003283b3936ab0af27c372a614c
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Mar 25 20:05:31 2021 -0400
[NO ISSUE][*DB] Update Dataverse canonical form
Update canonical dataverse name to use / as a part separator
*NOTE* this breaks metadata compatibility with existing multi-part
dataverse names, as the canonical format is stored in metadata
Change-Id: Ifc7d7fe5d7ce9a922371c1a9c6685d7a5dc64c33
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10704
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../special_chars_2/special_chars_2.2.adm | 6 +-
.../asterix/common/metadata/DataverseName.java | 74 ++++------------------
.../asterix/common/metadata/DataverseNameTest.java | 25 ++++----
3 files changed, 28 insertions(+), 77 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
index 9abda4f..7edd3eb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
@@ -1,6 +1,6 @@
{ "CanonicalName": "A", "DisplayName": "A", "NameParts": [ "A" ] }
-{ "CanonicalName": "B.C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
-{ "CanonicalName": "C@.D@.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
+{ "CanonicalName": "B/C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
+{ "CanonicalName": "C.D.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
{ "CanonicalName": "Default", "DisplayName": "Default", "NameParts": [ "Default" ] }
{ "CanonicalName": "Metadata", "DisplayName": "Metadata", "NameParts": [ "Metadata" ] }
-{ "CanonicalName": "a-A.b_B.c$C.z@.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
+{ "CanonicalName": "a-A/b_B/c$C/z.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
index b8124a2..f943dea 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DataverseName.java
@@ -36,11 +36,10 @@ import org.apache.commons.lang3.StringUtils;
* <p>
* Each dataverse name can be encoded into a single string (called a canonical form) by
* {@link #getCanonicalForm()} and decoded back from it with {@link #createFromCanonicalForm(String)}.
- * The canonical form encoding concatenates name parts together with {@link #CANONICAL_FORM_SEPARATOR_CHAR '.'}
- * character. The {@link #CANONICAL_FORM_ESCAPE_CHAR '@'} character is used to escape
- * {@link #CANONICAL_FORM_SEPARATOR_CHAR '.'} and itself in each name part prior to concatenation.
+ * The canonical form encoding concatenates name parts together with {@link #CANONICAL_FORM_SEPARATOR_CHAR '/'}
+ * character.
* <p>
- * E.g. the canonical form for a dataverse name {@code ["a", "b", "c"]} is {@code "a.b.c"}
+ * E.g. the canonical form for a dataverse name {@code ["a", "b", "c"]} is {@code "a/b/c"}
* <p>
* {@link #toString()} returns a display form which is suitable for error messages,
* and is a valid SQL++ multi-part identifier parsable by {@code IParser#parseMultipartIdentifier()}
@@ -59,11 +58,9 @@ import org.apache.commons.lang3.StringUtils;
*/
public final class DataverseName implements Serializable, Comparable<DataverseName> {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
- public static final char CANONICAL_FORM_SEPARATOR_CHAR = '.';
-
- private static final char CANONICAL_FORM_ESCAPE_CHAR = '@';
+ public static final char CANONICAL_FORM_SEPARATOR_CHAR = '/';
public static final char DISPLAY_FORM_SEPARATOR_CHAR = '.';
@@ -72,7 +69,7 @@ public final class DataverseName implements Serializable, Comparable<DataverseNa
private static final char DISPLAY_FORM_ESCAPE_CHAR = '\\';
private static final char[] CANONICAL_FORM_SEPARATOR_AND_ESCAPE_CHARS =
- new char[] { CANONICAL_FORM_SEPARATOR_CHAR, CANONICAL_FORM_ESCAPE_CHAR };
+ new char[] { CANONICAL_FORM_SEPARATOR_CHAR };
private final boolean isMultiPart;
@@ -282,13 +279,7 @@ public final class DataverseName implements Serializable, Comparable<DataverseNa
}
private static void encodePartIntoCanonicalForm(String part, StringBuilder out) {
- for (int i = 0, ln = part.length(); i < ln; i++) {
- char c = part.charAt(i);
- if (c == CANONICAL_FORM_SEPARATOR_CHAR || c == CANONICAL_FORM_ESCAPE_CHAR) {
- out.append(CANONICAL_FORM_ESCAPE_CHAR);
- }
- out.append(c);
- }
+ out.append(part);
}
private static <T> void decodeCanonicalForm(String canonicalForm, BiConsumer<CharSequence, T> partConsumer,
@@ -297,18 +288,11 @@ public final class DataverseName implements Serializable, Comparable<DataverseNa
StringBuilder sb = new StringBuilder(ln);
for (int i = 0; i < ln; i++) {
char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- partConsumer.accept(sb, partConsumerArg);
- sb.setLength(0);
- break;
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- c = canonicalForm.charAt(i);
- // fall through to 'default'
- default:
- sb.append(c);
- break;
+ if (c == CANONICAL_FORM_SEPARATOR_CHAR) {
+ partConsumer.accept(sb, partConsumerArg);
+ sb.setLength(0);
+ } else {
+ sb.append(c);
}
}
if (sb.length() > 0) {
@@ -318,41 +302,11 @@ public final class DataverseName implements Serializable, Comparable<DataverseNa
// optimization for a single part name
private static String decodeSinglePartNameFromCanonicalForm(String canonicalForm) {
- if (canonicalForm.indexOf(CANONICAL_FORM_ESCAPE_CHAR) < 0) {
- // no escaping was done
- return canonicalForm;
- }
-
- StringBuilder singlePart = new StringBuilder(canonicalForm.length());
- for (int i = 0, ln = canonicalForm.length(); i < ln; i++) {
- char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- throw new IllegalStateException(canonicalForm); // should never happen
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- c = canonicalForm.charAt(i);
- // fall through to 'default'
- default:
- singlePart.append(c);
- break;
- }
- }
- return singlePart.toString();
+ return canonicalForm;
}
private static boolean isMultiPartCanonicalForm(String canonicalForm) {
- for (int i = 0, ln = canonicalForm.length(); i < ln; i++) {
- char c = canonicalForm.charAt(i);
- switch (c) {
- case CANONICAL_FORM_SEPARATOR_CHAR:
- return true;
- case CANONICAL_FORM_ESCAPE_CHAR:
- i++;
- break;
- }
- }
- return false;
+ return canonicalForm.indexOf(CANONICAL_FORM_SEPARATOR_CHAR) != -1;
}
private static void addPartToCollection(CharSequence part, Collection<? super String> out) {
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
index 2f0dff5..75b3989 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/common/metadata/DataverseNameTest.java
@@ -51,9 +51,7 @@ public class DataverseNameTest {
private static final List<String> TEST_BUILTIN_DATAVERSE_INVALID_NAME_PARAMS = Arrays.asList(
// separator character is not allowed
- "a.b",
- // escape character is not allowed
- "c@d");
+ "a/b");
private static final List<Triple<String, String, String>> TEST_SINGLE_PART_NAME_PARAMS = Arrays.asList(
// <1-part-name, canonical-form, display-form>
@@ -63,30 +61,30 @@ public class DataverseNameTest {
// letters and digits
new Triple<>("aA09", "aA09", "aA09"),
// with canonical form escape character
- new Triple<>("a@b", "a@@b", "`a@b`"),
+ new Triple<>("a@b", "a@b", "`a@b`"),
// with canonical form separator character
- new Triple<>("a.b", "a@.b", "`a.b`"),
+ new Triple<>("a.b", "a.b", "`a.b`"),
// with canonical form escape and separator characters
- new Triple<>("a@.b", "a@@@.b", "`a@.b`"),
+ new Triple<>("a@.b", "a@.b", "`a@.b`"),
// with display form escape character
new Triple<>("a\\b", "a\\b", "`a\\\\b`"));
private static final List<Triple<List<String>, String, String>> TEST_MULTI_PART_NAME_PARAMS = Arrays.asList(
// <multi-part-name, canonical-form, display-form>
- new Triple<>(Arrays.asList("aa", "bb", "cc"), "aa.bb.cc", "aa.bb.cc"),
+ new Triple<>(Arrays.asList("aa", "bb", "cc"), "aa/bb/cc", "aa.bb.cc"),
// mixed case letters, digits
- new Triple<>(Arrays.asList("az", "AZ", "a09Z"), "az.AZ.a09Z", "az.AZ.a09Z"),
+ new Triple<>(Arrays.asList("az", "AZ", "a09Z"), "az/AZ/a09Z", "az.AZ.a09Z"),
// with canonical form escape character
- new Triple<>(Arrays.asList("a@a@", "@b@b", "@c@c"), "a@@a@@.@@b@@b.@@c@@c", "`a@a@`.`@b@b`.`@c@c`"),
+ new Triple<>(Arrays.asList("a@a@", "@b@b", "@c@c"), "a@a@/@b@b/@c@c", "`a@a@`.`@b@b`.`@c@c`"),
// with canonical form separator character
- new Triple<>(Arrays.asList("a.a.", ".b.b.", ".c.c"), "a@.a@..@.b@.b@..@.c@.c", "`a.a.`.`.b.b.`.`.c.c`"),
+ new Triple<>(Arrays.asList("a.a.", ".b.b.", ".c.c"), "a.a./.b.b./.c.c", "`a.a.`.`.b.b.`.`.c.c`"),
// with canonical form escape and separator characters
- new Triple<>(Arrays.asList("a@a.", "@b.b@", ".c@c"), "a@@a@..@@b@.b@@.@.c@@c", "`a@a.`.`@b.b@`.`.c@c`"),
+ new Triple<>(Arrays.asList("a@a.", "@b.b@", ".c@c"), "a@a./@b.b@/.c@c", "`a@a.`.`@b.b@`.`.c@c`"),
// with canonical form escape and separator characters repeated
- new Triple<>(Arrays.asList("a@@a..", "@@b..b@@", "..c@@c"), "a@@@@a@.@..@@@@b@.@.b@@@@.@.@.c@@@@c",
+ new Triple<>(Arrays.asList("a@@a..", "@@b..b@@", "..c@@c"), "a@@a../@@b..b@@/..c@@c",
"`a@@a..`.`@@b..b@@`.`..c@@c`"),
// with display form escape character
- new Triple<>(Arrays.asList("a\\b", "c\\d"), "a\\b.c\\d", "`a\\\\b`.`c\\\\d`"));
+ new Triple<>(Arrays.asList("a\\b", "c\\d"), "a\\b/c\\d", "`a\\\\b`.`c\\\\d`"));
@Test
public void testBuiltinDataverseName() throws Exception {
@@ -220,7 +218,6 @@ public class DataverseNameTest {
testRuntimeException(() -> DataverseName.createBuiltinDataverseName(null), NullPointerException.class);
testRuntimeException(() -> DataverseName.createFromCanonicalForm(null), NullPointerException.class);
testRuntimeException(() -> DataverseName.create(Collections.singletonList(null)), NullPointerException.class);
- testRuntimeException(() -> DataverseName.create(Arrays.asList(null, null)), NullPointerException.class);
// 3. IndexOutOfBoundsException
testRuntimeException(() -> DataverseName.create(Collections.emptyList(), 0, 1),
IndexOutOfBoundsException.class);
[asterixdb] 17/25: [NO ISSUE][*DB][TXN] Avoid abort log on empty
txn, reduce recovery logging
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 104d46a8cd11dad14ed6ff8f6a074b1171afb09b
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Apr 4 11:34:35 2021 -0400
[NO ISSUE][*DB][TXN] Avoid abort log on empty txn, reduce recovery logging
Change-Id: I91c01f34e38df7d2398e5383b5d39632e64f0e7f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10884
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 19 ++++++++-----------
.../service/transaction/TransactionManager.java | 12 ++++++++----
2 files changed, 16 insertions(+), 15 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 1461ef4..65cb36a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -599,22 +599,19 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
throw new ACIDException(e);
}
long lastLSN = txnContext.getLastLSN();
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("rollbacking transaction log records from " + firstLSN + " to " + lastLSN);
- }
+ boolean infoEnabled = LOGGER.isInfoEnabled();
// check if the transaction actually wrote some logs.
if (firstLSN == TransactionManagementConstants.LogManagerConstants.TERMINAL_LSN || firstLSN > lastLSN) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("no need to roll back as there were no operations by the txn " + txnContext.getTxnId());
+ if (infoEnabled) {
+ LOGGER.info("no need to rollback as there were no operations by " + txnContext.getTxnId());
}
return;
}
-
- // While reading log records from firstLsn to lastLsn, collect uncommitted txn's Lsns
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("collecting loser transaction's LSNs from " + firstLSN + " to " + lastLSN);
+ if (infoEnabled) {
+ LOGGER.info("rolling back transaction log records from " + firstLSN + " to " + lastLSN + " for "
+ + txnContext.getTxnId());
}
-
+ // While reading log records from firstLsn to lastLsn, collect uncommitted txn's Lsns
Map<TxnEntityId, List<Long>> jobLoserEntity2LSNsMap = new HashMap<>();
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, false);
int updateLogCount = 0;
@@ -722,7 +719,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
}
}
- if (LOGGER.isInfoEnabled()) {
+ if (infoEnabled) {
LOGGER.info("undone loser transaction's effect");
LOGGER.info("[RecoveryManager's rollback log count] update/entityCommit/undo:" + updateLogCount + "/"
+ entityCommitLogCount + "/" + undoCount);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
index c218dec..ee65962 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.transaction.management.service.transaction;
+import static org.apache.asterix.transaction.management.service.transaction.TransactionManagementConstants.LogManagerConstants.TERMINAL_LSN;
+
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;
@@ -101,10 +103,12 @@ public class TransactionManager implements ITransactionManager, ILifeCycleCompon
final ITransactionContext txnCtx = getTransactionContext(txnId);
try {
if (txnCtx.isWriteTxn()) {
- LogRecord logRecord = new LogRecord();
- TransactionUtil.formJobTerminateLogRecord(txnCtx, logRecord, false);
- txnSubsystem.getLogManager().log(logRecord);
- txnSubsystem.getCheckpointManager().secure(txnId);
+ if (txnCtx.getFirstLSN() != TERMINAL_LSN) {
+ LogRecord logRecord = new LogRecord();
+ TransactionUtil.formJobTerminateLogRecord(txnCtx, logRecord, false);
+ txnSubsystem.getLogManager().log(logRecord);
+ txnSubsystem.getCheckpointManager().secure(txnId);
+ }
txnSubsystem.getRecoveryManager().rollbackTransaction(txnCtx);
txnCtx.setTxnState(ITransactionManager.ABORTED);
}
[asterixdb] 08/25: [NO ISSUE][EXT] Avoid duplicate open for streams
+ minor refactoring
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 27792084826c46d5980c0aafa43affc580e2c576
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Tue Mar 30 23:56:35 2021 +0300
[NO ISSUE][EXT] Avoid duplicate open for streams + minor refactoring
Change-Id: I405e84a30ee67b176c3389db6fd026c408ae1685
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10783
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../abstracts/AbstractExternalInputStreamFactory.java | 13 +++++++++++--
.../input/record/reader/aws/AwsS3InputStream.java | 11 ++++++-----
.../input/record/reader/aws/AwsS3InputStreamFactory.java | 15 ++++-----------
.../input/record/reader/azure/AzureBlobInputStream.java | 12 ++++++------
.../record/reader/azure/AzureBlobInputStreamFactory.java | 15 ++++-----------
.../asterix/external/util/ExternalDataConstants.java | 4 +---
.../apache/asterix/external/util/ExternalDataUtils.java | 4 ++--
7 files changed, 34 insertions(+), 40 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
index ca55b6f..0b215a0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
@@ -30,6 +30,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
@@ -69,9 +70,17 @@ public abstract class AbstractExternalInputStreamFactory implements IInputStream
return partitionConstraint;
}
+ protected int getPartitionsCount() {
+ return getPartitionConstraint().getLocations().length;
+ }
+
@Override
- public abstract void configure(IServiceContext ctx, Map<String, String> configuration,
- IWarningCollector warningCollector) throws AlgebricksException;
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
+ throws AlgebricksException {
+ this.configuration = configuration;
+ this.partitionConstraint =
+ ((ICcApplicationContext) ctx.getApplicationContext()).getClusterStateManager().getClusterLocations();
+ }
/**
* Finds the smallest workload and returns it
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index e3e53d5..48035f3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -42,15 +42,17 @@ import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
public class AwsS3InputStream extends AbstractExternalInputStream {
private final S3Client s3Client;
+ private final String bucket;
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
this.s3Client = buildAwsS3Client(configuration);
+ this.bucket = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@Override
protected boolean getInputStream() throws IOException {
- String bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(filePaths.get(nextFileIndex)).build();
@@ -67,11 +69,10 @@ public class AwsS3InputStream extends AbstractExternalInputStream {
}
// Use gzip stream if needed
- String filename = filePaths.get(nextFileIndex).toLowerCase();
- if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(s3Client.getObject(getObjectRequest), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ String lowerCaseFileName = fileName.toLowerCase();
+ if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
+ in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
-
return true;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 5bab888..a1c577a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -18,19 +18,17 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -61,8 +59,7 @@ public class AwsS3InputStreamFactory extends AbstractExternalInputStreamFactory
@Override
public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException {
- this.configuration = configuration;
- ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext();
+ super.configure(ctx, configuration, warningCollector);
// Ensure the validity of include/exclude
ExternalDataUtils.validateIncludeExclude(configuration);
@@ -70,7 +67,7 @@ public class AwsS3InputStreamFactory extends AbstractExternalInputStreamFactory
// Prepare to retrieve the objects
List<S3Object> filesOnly;
- String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
S3Client s3Client = ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
try {
@@ -101,12 +98,8 @@ public class AwsS3InputStreamFactory extends AbstractExternalInputStreamFactory
warningCollector.warn(warning);
}
- // Partition constraints
- partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations();
- int partitionsCount = partitionConstraint.getLocations().length;
-
// Distribute work load amongst the partitions
- distributeWorkLoad(filesOnly, partitionsCount);
+ distributeWorkLoad(filesOnly, getPartitionsCount());
}
/**
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
index 358c412..3fb3395 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStream.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.azure;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -43,15 +41,17 @@ import com.azure.storage.blob.models.BlobStorageException;
public class AzureBlobInputStream extends AbstractExternalInputStream {
private final BlobServiceClient client;
+ private final String container;
public AzureBlobInputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
this.client = buildAzureClient(configuration);
+ this.container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@Override
protected boolean getInputStream() throws IOException {
- String container = configuration.get(AzureBlob.CONTAINER_NAME_FIELD_NAME);
+ String fileName = filePaths.get(nextFileIndex);
BlobContainerClient blobContainerClient;
BlobClient blob;
try {
@@ -60,9 +60,9 @@ public class AzureBlobInputStream extends AbstractExternalInputStream {
in = blob.openInputStream();
// Use gzip stream if needed
- String filename = filePaths.get(nextFileIndex).toLowerCase();
- if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(in = blob.openInputStream(), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ String lowerCaseFileName = fileName.toLowerCase();
+ if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
+ in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
index 167e22a..ca064b1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
@@ -18,19 +18,17 @@
*/
package org.apache.asterix.external.input.record.reader.azure;
-import static org.apache.asterix.external.util.ExternalDataConstants.*;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
@@ -56,10 +54,9 @@ public class AzureBlobInputStreamFactory extends AbstractExternalInputStreamFact
@Override
public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException {
- this.configuration = configuration;
- ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext();
+ super.configure(ctx, configuration, warningCollector);
- String container = configuration.get(AzureBlob.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
List<BlobItem> filesOnly = new ArrayList<>();
@@ -87,12 +84,8 @@ public class AzureBlobInputStreamFactory extends AbstractExternalInputStreamFact
warningCollector.warn(warning);
}
- // Partition constraints
- partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations();
- int partitionsCount = partitionConstraint.getLocations().length;
-
// Distribute work load amongst the partitions
- distributeWorkLoad(filesOnly, partitionsCount);
+ distributeWorkLoad(filesOnly, getPartitionsCount());
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index fd5b269..45d15df 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -288,6 +288,7 @@ public class ExternalDataConstants {
public static final String INVALID_VAL = "invalid value";
public static final String DEFINITION_FIELD_NAME = "definition";
+ public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static class AwsS3 {
private AwsS3() {
@@ -298,7 +299,6 @@ public class ExternalDataConstants {
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
}
@@ -307,8 +307,6 @@ public class ExternalDataConstants {
throw new AssertionError("do not instantiate");
}
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
- public static final String DEFINITION_FIELD_NAME = "definition";
public static final String CONNECTION_STRING_FIELD_NAME = "connectionString";
public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 3ac1116..d1bbe89 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -776,7 +776,7 @@ public class ExternalDataUtils {
S3Client s3Client = buildAwsS3Client(configuration);;
S3Response response;
boolean useOldApi = false;
- String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
String prefix = getPrefix(configuration);
try {
@@ -943,7 +943,7 @@ public class ExternalDataUtils {
// Check if the bucket is present
BlobServiceClient blobServiceClient;
try {
- String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
+ String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
blobServiceClient = buildAzureClient(configuration);
BlobContainerClient blobContainer = blobServiceClient.getBlobContainerClient(container);
[asterixdb] 01/25: [NO ISSUE][EXT] Make read buffer size
configurable
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c74bdc8d3100a8724b24bffa6ff25d01d8b339c1
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Sun Mar 21 21:31:28 2021 -0700
[NO ISSUE][EXT] Make read buffer size configurable
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Make read buffer size configurable for external datasets
using the compiler property "compiler.externalscanmemory"
Change-Id: Ieb32b347b02fb800f13947f53267faa4837f3248
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10624
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../optimizer/rules/ConstantFoldingRule.java | 3 +-
.../apache/asterix/api/common/APIFramework.java | 11 ++--
.../external_dataset.000.ddl.sqlpp | 66 ++++++++++++++++++++++
.../external_dataset.001.query.sqlpp | 13 +----
.../external_dataset.002.query.sqlpp | 13 +----
.../external_dataset.003.query.sqlpp | 13 +----
.../external_dataset.004.query.sqlpp | 13 +----
.../external_dataset.005.query.sqlpp | 13 +----
.../external_dataset.006.query.sqlpp | 13 +----
.../external_dataset.999.ddl.sqlpp | 12 +---
.../s3/custom-buffer-size/external_dataset.001.adm | 1 +
.../s3/custom-buffer-size/external_dataset.002.adm | 1 +
.../s3/custom-buffer-size/external_dataset.003.adm | 1 +
.../s3/custom-buffer-size/external_dataset.004.adm | 25 ++++++++
.../s3/custom-buffer-size/external_dataset.005.adm | 1 +
.../s3/custom-buffer-size/external_dataset.006.adm | 25 ++++++++
.../runtimets/testsuite_external_dataset.xml | 5 ++
.../asterix/common/config/CompilerProperties.java | 12 +++-
.../common/config/OptimizationConfUtil.java | 15 +++++
.../input/record/reader/aws/AwsS3InputStream.java | 5 +-
.../record/reader/stream/StreamRecordReader.java | 6 +-
.../input/stream/AsterixInputStreamReader.java | 16 +++---
.../external/util/ExternalDataConstants.java | 1 +
.../asterix/external/util/ExternalDataUtils.java | 6 ++
.../metadata/declared/DatasetDataSource.java | 13 ++++-
.../api/HeuristicCompilerFactoryBuilder.java | 2 +-
.../algebricks/core/config/AlgebricksConfig.java | 3 +
.../algebricks/core/jobgen/impl/JobGenContext.java | 9 ++-
.../rewriter/base/PhysicalOptimizationConfig.java | 9 +++
29 files changed, 231 insertions(+), 95 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 47e0373..6f359c5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -83,6 +83,7 @@ import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVis
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
@@ -146,7 +147,7 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule {
BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null,
new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null,
- NoOpWarningCollector.INSTANCE, 0);
+ NoOpWarningCollector.INSTANCE, 0, new PhysicalOptimizationConfig());
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index d573b52..c863080 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -134,11 +134,12 @@ public class APIFramework {
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME,
- FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION,
- FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS,
- SqlppQueryRewriter.INLINE_WITH_OPTION, SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION,
- "hash_merge", "output-record-type", AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
+ StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
+ FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
+ SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
+ AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION,
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION);
private final IRewriterFactory rewriterFactory;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000..22a30ca
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test1 if exists;
+create external dataset test1(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json"));
+
+drop dataset test2 if exists;
+create external dataset test2(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines/json"),
+("format"="json"));
+
+drop dataset test3 if exists;
+create external dataset test3(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-arrays/json"),
+("format"="json"));
+
+drop dataset test4 if exists;
+create external dataset test4(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-nested-objects/json"),
+("format"="json"));
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp
index 15bb54b..7c1708a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "8KB";
+select count(*) `count` from test1;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp
index 15bb54b..d122f42 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "16KB";
+select count(*) `count` from test2;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp
index 15bb54b..552f943 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "32KB";
+select count(*) `count` from test3;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp
index 15bb54b..f30af80 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "8KB";
+select value test3 from test3 order by id;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp
index 15bb54b..af6aff0 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "16KB";
+select count(*) `count` from test4;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp
index 15bb54b..854ac80 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp
@@ -16,15 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+use test;
+set `compiler.externalscanmemory` "10KB";
+select value test4 from test4 order by id;
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp
similarity index 63%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp
index 15bb54b..548e632 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp
@@ -16,15 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.config;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class AlgebricksConfig {
-
- public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
- public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
- public static final boolean SORT_PARALLEL = true;
-}
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm
new file mode 100644
index 0000000..5db606c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm
@@ -0,0 +1 @@
+{ "count": 25 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm
new file mode 100644
index 0000000..7660e7e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm
@@ -0,0 +1,25 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm
new file mode 100644
index 0000000..5db606c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm
@@ -0,0 +1 @@
+{ "count": 25 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm
new file mode 100644
index 0000000..7643986
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm
@@ -0,0 +1,25 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 9cded35..096e511 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -19,6 +19,11 @@
!-->
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
<test-group name="external-dataset">
+ <test-case FilePath="external-dataset/aws/s3/">
+ <compilation-unit name="custom-buffer-size">
+ <output-dir compare="Text">custom-buffer-size</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="aws/s3/json/json">
<output-dir compare="Text">aws/s3/json/json</output-dir>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 8860495..652f390 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -56,6 +56,10 @@ public class CompilerProperties extends AbstractProperties {
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
"The memory budget (in bytes) for an inverted-index-search operator instance in a partition"),
+ COMPILER_EXTERNALSCANMEMORY(
+ INTEGER_BYTE_UNIT,
+ StorageUtil.getIntSizeInBytes(4, KILOBYTE),
+ "The memory budget (in bytes) for an external scan operator instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@ -110,7 +114,7 @@ public class CompilerProperties extends AbstractProperties {
@Override
public boolean hidden() {
- return this == COMPILER_STRINGOFFSET;
+ return this == COMPILER_STRINGOFFSET || this == COMPILER_EXTERNALSCANMEMORY;
}
}
@@ -130,6 +134,8 @@ public class CompilerProperties extends AbstractProperties {
public static final String COMPILER_SORT_SAMPLES_KEY = Option.COMPILER_SORT_SAMPLES.ini();
+ public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@ -176,4 +182,8 @@ public class CompilerProperties extends AbstractProperties {
public int getSortSamples() {
return accessor.getInt(Option.COMPILER_SORT_SAMPLES);
}
+
+ public int getExternalScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 23fdcac..22fb13b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -59,6 +59,9 @@ public class OptimizationConfUtil {
int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc);
int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc);
boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig);
+ int externalScanBufferSize = getExternalScanBufferSize(
+ (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
+ compilerProperties.getExternalScanMemorySize(), sourceLoc);
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
@@ -69,10 +72,22 @@ public class OptimizationConfUtil {
physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
physOptConf.setSortParallel(fullParallelSort);
physOptConf.setSortSamples(sortNumSamples);
+ physOptConf.setExternalScanBufferSize(externalScanBufferSize);
return physOptConf;
}
+ private static int getExternalScanBufferSize(String externalScanMemorySizeParameter,
+ int compilerExternalScanMemorySize, SourceLocation sourceLoc) throws AsterixException {
+ IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT;
+ try {
+ return externalScanMemorySizeParameter != null ? intByteParser.parse(externalScanMemorySizeParameter)
+ : compilerExternalScanMemorySize;
+ } catch (IllegalArgumentException e) {
+ throw AsterixException.create(ErrorCode.COMPILATION_ERROR, sourceLoc, e.getMessage());
+ }
+ }
+
public static int getSortNumFrames(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig,
SourceLocation sourceLoc) throws AlgebricksException {
return getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY,
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 9e10e6a..f558820 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -29,7 +29,6 @@ import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.stream.AbstractMultipleInputStream;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -48,6 +47,7 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
// Configuration
private final Map<String, String> configuration;
+ private final int bufferSize;
private final S3Client s3Client;
@@ -59,6 +59,7 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
this.configuration = configuration;
this.filePaths = filePaths;
this.s3Client = buildAwsS3Client(configuration);
+ this.bufferSize = ExternalDataUtils.getOrDefaultBufferSize(configuration);
}
@Override
@@ -101,7 +102,7 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
// Use gzip stream if needed
String filename = filePaths.get(nextFileIndex).toLowerCase();
if (filename.endsWith(".gz") || filename.endsWith(".gzip")) {
- in = new GZIPInputStream(s3Client.getObject(getObjectRequest), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ in = new GZIPInputStream(s3Client.getObject(getObjectRequest), bufferSize);
}
// Current file ready, point to the next file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
index cb16de5..1e003a1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReader.java
@@ -33,7 +33,6 @@ import org.apache.asterix.external.api.IStreamNotificationHandler;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.stream.AsterixInputStreamReader;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -51,9 +50,10 @@ public abstract class StreamRecordReader implements IRecordReader<char[]>, IStre
private Supplier<String> previousDataSourceName = EMPTY_STRING;
public void configure(AsterixInputStream inputStream, Map<String, String> config) {
- this.reader = new AsterixInputStreamReader(inputStream);
+ int bufferSize = ExternalDataUtils.getOrDefaultBufferSize(config);
+ this.reader = new AsterixInputStreamReader(inputStream, bufferSize);
record = new CharArrayRecord();
- inputBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ inputBuffer = new char[bufferSize];
if (!ExternalDataUtils.isTrue(config, KEY_REDACT_WARNINGS)) {
this.dataSourceName = reader::getStreamName;
this.previousDataSourceName = reader::getPreviousStreamName;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index 4e963e4..ddbd350 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -27,21 +27,23 @@ import java.nio.charset.StandardCharsets;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AsterixInputStreamReader extends Reader {
- private AsterixInputStream in;
- private byte[] bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- private ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
- private CharBuffer charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
- private CharsetDecoder decoder;
+ private final AsterixInputStream in;
+ private final byte[] bytes;
+ private final ByteBuffer byteBuffer;
+ private final CharBuffer charBuffer;
+ private final CharsetDecoder decoder;
private boolean done = false;
private boolean remaining = false;
- public AsterixInputStreamReader(AsterixInputStream in) {
+ public AsterixInputStreamReader(AsterixInputStream in, int bufferSize) {
this.in = in;
+ this.bytes = new byte[bufferSize];
+ this.byteBuffer = ByteBuffer.wrap(bytes);
+ this.charBuffer = CharBuffer.allocate(bufferSize);
this.decoder = StandardCharsets.UTF_8.newDecoder();
this.byteBuffer.flip();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 53306c5..1911083 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -117,6 +117,7 @@ public class ExternalDataConstants {
// a string representing the NULL value
public static final String KEY_NULL_STR = "null";
public static final String KEY_REDACT_WARNINGS = "redact-warnings";
+ public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
/**
* Keys for adapter name
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index c0a7a4d..bc9a038 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -20,6 +20,7 @@ package org.apache.asterix.external.util;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
@@ -90,6 +91,11 @@ public class ExternalDataUtils {
private ExternalDataUtils() {
}
+ public static int getOrDefaultBufferSize(Map<String, String> configuration) {
+ String bufferSize = configuration.get(KEY_EXTERNAL_SCAN_BUFFER_SIZE);
+ return bufferSize != null ? Integer.parseInt(bufferSize) : ExternalDataConstants.DEFAULT_BUFFER_SIZE;
+ }
+
// Get a delimiter from the given configuration
public static char validateGetDelimiter(Map<String, String> configuration) throws HyracksDataException {
return validateCharOrDefault(configuration, KEY_DELIMITER, ExternalDataConstants.DEFAULT_DELIMITER.charAt(0));
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 71d389e..715092a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -18,7 +18,10 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
+
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -42,6 +45,7 @@ import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
@@ -111,9 +115,12 @@ public class DatasetDataSource extends DataSource {
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
- ITypedAdapterFactory adapterFactory =
- metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(),
- edd.getProperties(), (ARecordType) itemType, null, context.getWarningCollector());
+ PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+ int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
+ Map<String, String> properties = edd.getProperties();
+ properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
+ ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
+ edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
case INTERNAL:
DataSourceId id = getId();
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index 0fd107f..b96d657 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -105,7 +105,7 @@ public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuil
normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer,
oc, expressionEvalSizeComputer, partialAggregationTypeComputer,
predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(),
- clusterLocations, warningCollector, maxWarnings);
+ clusterLocations, warningCollector, maxWarnings, physicalOptimizationConfig);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 15bb54b..7630a13 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.algebricks.core.config;
+import org.apache.hyracks.util.StorageUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -26,5 +27,7 @@ public class AlgebricksConfig {
public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
public static final int SORT_SAMPLES = 100;
+ public static final int EXTERNAL_SCAN_BUFFER_SIZE =
+ StorageUtil.getIntSizeInBytes(4, StorageUtil.StorageUnit.KILOBYTE);
public static final boolean SORT_PARALLEL = true;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
index 4bc5689..3c03b18 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
@@ -34,6 +34,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir
import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import org.apache.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
@@ -68,6 +69,7 @@ public class JobGenContext {
private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
private final IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
private final int frameSize;
+ private final PhysicalOptimizationConfig physicalOptimizationConfig;
private AlgebricksAbsolutePartitionConstraint clusterLocations;
private int varCounter;
private final ITypingContext typingContext;
@@ -88,7 +90,7 @@ public class JobGenContext {
IPartialAggregationTypeComputer partialAggregationTypeComputer,
IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize,
AlgebricksAbsolutePartitionConstraint clusterLocations, IWarningCollector warningCollector,
- long maxWarnings) {
+ long maxWarnings, PhysicalOptimizationConfig physicalOptimizationConfig) {
this.outerFlowSchema = outerFlowSchema;
this.metadataProvider = metadataProvider;
this.appContext = appContext;
@@ -110,6 +112,7 @@ public class JobGenContext {
this.partialAggregationTypeComputer = partialAggregationTypeComputer;
this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
this.frameSize = frameSize;
+ this.physicalOptimizationConfig = physicalOptimizationConfig;
this.varCounter = 0;
this.warningCollector = warningCollector;
this.maxWarnings = maxWarnings;
@@ -220,4 +223,8 @@ public class JobGenContext {
public long getMaxWarnings() {
return maxWarnings;
}
+
+ public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+ return physicalOptimizationConfig;
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index 598497c..93f5d74 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -39,6 +39,7 @@ public class PhysicalOptimizationConfig {
private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
private static final String SORT_PARALLEL = "SORT_PARALLEL";
private static final String SORT_SAMPLES = "SORT_SAMPLES";
+ private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
private Properties properties = new Properties();
@@ -172,6 +173,14 @@ public class PhysicalOptimizationConfig {
setInt(SORT_SAMPLES, sortSamples);
}
+ public int getExternalScanBufferSize() {
+ return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
+ }
+
+ public void setExternalScanBufferSize(int bufferSize) {
+ setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
[asterixdb] 10/25: [NO ISSUE][EXT]: Improve workload distribution
logic
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d3827662ad11193341a12d931bf2eb8188f07e59
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Mar 31 18:10:49 2021 +0300
[NO ISSUE][EXT]: Improve workload distribution logic
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Improve the workload distribution logic.
Change-Id: I191dc87850c1812b49831dd3c78bc7a22cc5b931
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10763
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../AbstractExternalInputStreamFactory.java | 21 -------
.../record/reader/aws/AwsS3InputStreamFactory.java | 13 +++-
.../reader/azure/AzureBlobInputStreamFactory.java | 15 +++--
.../input/record/reader/awss3/AwsS3Test.java | 73 ++++++++++++++++++++++
4 files changed, 94 insertions(+), 28 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
index 0b215a0..2a063bf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStreamFactory.java
@@ -82,27 +82,6 @@ public abstract class AbstractExternalInputStreamFactory implements IInputStream
((ICcApplicationContext) ctx.getApplicationContext()).getClusterStateManager().getClusterLocations();
}
- /**
- * Finds the smallest workload and returns it
- *
- * @return the smallest workload
- */
- protected PartitionWorkLoadBasedOnSize getSmallestWorkLoad() {
- PartitionWorkLoadBasedOnSize smallest = partitionWorkLoadsBasedOnSize.get(0);
- for (PartitionWorkLoadBasedOnSize partition : partitionWorkLoadsBasedOnSize) {
- // If the current total size is 0, add the file directly as this is a first time partition
- if (partition.getTotalSize() == 0) {
- smallest = partition;
- break;
- }
- if (partition.getTotalSize() < smallest.getTotalSize()) {
- smallest = partition;
- }
- }
-
- return smallest;
- }
-
protected IncludeExcludeMatcher getIncludeExcludeMatchers() throws CompilationException {
// Get and compile the patterns for include/exclude if provided
List<Matcher> includeMatchers = new ArrayList<>();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index a1c577a..747fcca 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -19,8 +19,10 @@
package org.apache.asterix.external.input.record.reader.aws;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
@@ -216,14 +218,19 @@ public class AwsS3InputStreamFactory extends AbstractExternalInputStreamFactory
* @param partitionsCount Partitions count
*/
private void distributeWorkLoad(List<S3Object> fileObjects, int partitionsCount) {
+ PriorityQueue<PartitionWorkLoadBasedOnSize> workloadQueue = new PriorityQueue<>(partitionsCount,
+ Comparator.comparingLong(PartitionWorkLoadBasedOnSize::getTotalSize));
+
// Prepare the workloads based on the number of partitions
for (int i = 0; i < partitionsCount; i++) {
- partitionWorkLoadsBasedOnSize.add(new PartitionWorkLoadBasedOnSize());
+ workloadQueue.add(new PartitionWorkLoadBasedOnSize());
}
for (S3Object object : fileObjects) {
- PartitionWorkLoadBasedOnSize smallest = getSmallestWorkLoad();
- smallest.addFilePath(object.key(), object.size());
+ PartitionWorkLoadBasedOnSize workload = workloadQueue.poll();
+ workload.addFilePath(object.key(), object.size());
+ workloadQueue.add(workload);
}
+ partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
index ca064b1..803694a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/azure/AzureBlobInputStreamFactory.java
@@ -19,8 +19,10 @@
package org.apache.asterix.external.input.record.reader.azure;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
@@ -131,14 +133,19 @@ public class AzureBlobInputStreamFactory extends AbstractExternalInputStreamFact
* @param partitionsCount Partitions count
*/
private void distributeWorkLoad(List<BlobItem> items, int partitionsCount) {
+ PriorityQueue<PartitionWorkLoadBasedOnSize> workloadQueue = new PriorityQueue<>(partitionsCount,
+ Comparator.comparingLong(PartitionWorkLoadBasedOnSize::getTotalSize));
+
// Prepare the workloads based on the number of partitions
for (int i = 0; i < partitionsCount; i++) {
- partitionWorkLoadsBasedOnSize.add(new PartitionWorkLoadBasedOnSize());
+ workloadQueue.add(new PartitionWorkLoadBasedOnSize());
}
- for (BlobItem item : items) {
- PartitionWorkLoadBasedOnSize smallest = getSmallestWorkLoad();
- smallest.addFilePath(item.getName(), item.getProperties().getContentLength());
+ for (BlobItem object : items) {
+ PartitionWorkLoadBasedOnSize workload = workloadQueue.poll();
+ workload.addFilePath(object.getName(), object.getProperties().getContentLength());
+ workloadQueue.add(workload);
}
+ partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
}
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
new file mode 100644
index 0000000..9cdeb16
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.awss3;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStreamFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public class AwsS3Test {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWorkloadDistribution() throws Exception {
+ AwsS3InputStreamFactory factory = new AwsS3InputStreamFactory();
+
+ List<S3Object> s3Objects = new ArrayList<>();
+ final int partitionsCount = 3;
+
+ // Create S3 objects, 9 objects, on 3 partitions, they should be 600 total size on each partition
+ S3Object.Builder builder = S3Object.builder();
+ s3Objects.add(builder.key("1.json").size(100L).build());
+ s3Objects.add(builder.key("2.json").size(100L).build());
+ s3Objects.add(builder.key("3.json").size(100L).build());
+ s3Objects.add(builder.key("4.json").size(200L).build());
+ s3Objects.add(builder.key("5.json").size(200L).build());
+ s3Objects.add(builder.key("6.json").size(200L).build());
+ s3Objects.add(builder.key("7.json").size(300L).build());
+ s3Objects.add(builder.key("8.json").size(300L).build());
+ s3Objects.add(builder.key("9.json").size(300L).build());
+
+ // invoke the distributeWorkLoad method
+ Method distributeWorkloadMethod =
+ AwsS3InputStreamFactory.class.getDeclaredMethod("distributeWorkLoad", List.class, int.class);
+ distributeWorkloadMethod.setAccessible(true);
+ distributeWorkloadMethod.invoke(factory, s3Objects, partitionsCount);
+
+ // get the partitionWorkLoadsBasedOnSize field and verify the result
+ Field distributeWorkloadField =
+ AwsS3InputStreamFactory.class.getSuperclass().getDeclaredField("partitionWorkLoadsBasedOnSize");
+ distributeWorkloadField.setAccessible(true);
+ List<AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize> workloads =
+ (List<AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize>) distributeWorkloadField
+ .get(factory);
+
+ for (AbstractExternalInputStreamFactory.PartitionWorkLoadBasedOnSize workload : workloads) {
+ Assert.assertEquals(workload.getTotalSize(), 600);
+ }
+ }
+}
[asterixdb] 03/25: [NO ISSUE][COMP] Improve subplan inlining
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6c5903ada976be29dd82dee327d99bdcb617c7a1
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Thu Mar 25 17:52:05 2021 -0700
[NO ISSUE][COMP] Improve subplan inlining
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Change EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables()
to use row_number() instead of create-query-uid() when generating a
key for a join that recovers the correlation
- Add "rewrite_internal_queryuid_pk" compiler option to completely
disable usage of create-query-uid() by the above method
- create-query-uid() should be invoked by RunningAggregateOperator instead
of Assign operator. Change compiler code and runtime to do that
- Make create-query-uid() private function
- Move side-effecting code out of
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables()
and make it a responsibility of the caller to compute primary
keys before invoking this method
Change-Id: Id57258211780897a3043f371d416cc56bb2f7e39
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10705
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../rules/SweepIllegalNonfunctionalFunctions.java | 3 -
.../subplan/InlineAllNtsInSubplanVisitor.java | 84 +-
...InlineSubplanInputForNestedTupleSourceRule.java | 48 +-
.../rules/subplan/SubplanFlatteningUtil.java | 51 +-
.../rules/util/EquivalenceClassUtils.java | 84 +-
.../apache/asterix/api/common/APIFramework.java | 4 +-
.../queries/subquery/in_let_8.sqlpp} | 29 +-
.../queries/subquery/query-ASTERIXDB-2845.sqlpp | 2 +-
.../optimizerts/results/ASTERIXDB-2402.plan | 28 +-
.../results/leftouterjoin/loj-03-no-listify.plan | 134 +-
.../optimizerts/results/query-issue562_ps.plan | 58 +-
.../optimizerts/results/query_issue849.plan | 8 +-
.../optimizerts/results/subquery/exists.plan | 2 +-
.../optimizerts/results/subquery/exists_ps.plan | 4 +-
.../optimizerts/results/subquery/in_let_3.plan | 4 +-
.../optimizerts/results/subquery/in_let_4.plan | 4 +-
.../optimizerts/results/subquery/in_let_5.plan | 4 +-
.../subquery/{in_let_3.plan => in_let_8.plan} | 60 +-
.../optimizerts/results/subquery/not_exists.plan | 2 +-
.../results/subquery/not_exists_ps.plan | 4 +-
.../results/subquery/query-ASTERIXDB-2845.plan | 145 +-
.../tpcds/query-ASTERIXDB-1581-correlated-2.plan | 399 +-
.../tpcds/query-ASTERIXDB-1581-correlated.plan | 407 +-
.../binary/query_id/query_id.1.ddl.sqlpp | 46 -
.../in_let/in_let.8.query.sqlpp} | 23 +-
.../results/binary/query_id/query_id.1.adm | 6005 --------------------
.../runtimets/results/subquery/in_let/in_let.8.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 -
.../asterix/om/functions/BuiltinFunctions.java | 2 +-
.../functions/CreateQueryUIDDescriptor.java | 32 +-
.../logical/RunningAggregateOperator.java | 5 +
.../algebra/util/OperatorManipulationUtil.java | 9 +-
32 files changed, 904 insertions(+), 6793 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 9bfa8fb..2509010 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -125,9 +125,6 @@ public class SweepIllegalNonfunctionalFunctions implements IAlgebraicRewriteRule
@Override
public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue());
- }
return null;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 02dc6f6..1b7b7d2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -47,6 +48,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -99,10 +101,11 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
* The visitor ensures that the variables used to correlate between the
* query plan rooted at <code>subplanInputOperator</code> are propagated
* to the operator being visited.
- *
- * ----------------------------------
+ * <p>
* Here is an abstract example.
+ * <p>
* The original query plan:
+ * <pre>
* --Op1
* --Subplan{
* --AggregateOp
@@ -112,9 +115,10 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
* }
* --InputOp
* .....
- *
+ * </pre>
* After we call NestedOp.accept(....) with this visitor. We will get an
* intermediate plan that looks like:
+ * <pre>
* --Op1
* --Subplan{
* --AggregateOp
@@ -125,9 +129,9 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
* }
* --InputOp
* .....
+ * </pre>
* The plan rooted at InputOp' is a deep copy of the plan rooted at InputOp
* with a different set of variables.
- *
*/
class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOperator, Void> {
// The optimization context.
@@ -186,17 +190,6 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
@Override
- public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
- return visitAggregateOperator(op);
- }
-
- @Override
- public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
- throws AlgebricksException {
- return visitAggregateOperator(op);
- }
-
- @Override
public ILogicalOperator visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg)
throws AlgebricksException {
return visitSingleInputOperator(op);
@@ -248,14 +241,38 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
@Override
+ public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
+ throws AlgebricksException {
+ return visitLimitOrRunningAggregateOperator(op, InlineAllNtsInSubplanVisitor::isCreateQueryUidRunningAggregate);
+ }
+
+ /*
+ * create-query-uid() doesn't require GROUP BY wrapping in the presence of correlated keys
+ */
+ private static boolean isCreateQueryUidRunningAggregate(RunningAggregateOperator ragg) {
+ return ragg.getExpressions().stream().map(Mutable::getValue)
+ .allMatch(expr -> expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL
+ && BuiltinFunctions.CREATE_QUERY_UID
+ .equals(((AbstractFunctionCallExpression) expr).getFunctionIdentifier()));
+ }
+
+ @Override
public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+ return visitLimitOrRunningAggregateOperator(op, null);
+ }
+
+ private <T extends ILogicalOperator> ILogicalOperator visitLimitOrRunningAggregateOperator(T op,
+ Predicate<T> noCorrelatedKeysHandlingTest) throws AlgebricksException {
// Processes its input operator.
visitSingleInputOperator(op);
if (correlatedKeyVars.isEmpty()) {
return op;
}
+ if (noCorrelatedKeysHandlingTest != null && noCorrelatedKeysHandlingTest.test(op)) {
+ return op;
+ }
- // Get live variables before limit.
+ // Get live variables before limit (or running aggregate) operator.
Set<LogicalVariable> inputLiveVars = new HashSet<LogicalVariable>();
VariableUtilities.getSubplanLocalLiveVariables(op.getInputs().get(0).getValue(), inputLiveVars);
@@ -267,8 +284,9 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
ILogicalOperator inputOp = op.getInputs().get(0).getValue();
assignOp.getInputs().add(new MutableObject<>(inputOp));
- // Rewrites limit to a group-by with limit as its nested operator.
- Pair<ILogicalOperator, LogicalVariable> gbyOpAndAggVar = wrapLimitInGroupBy(op, recordVar, inputLiveVars);
+ // Rewrites limit (or running aggregate) to a group-by with limit (or running aggregate) as its nested operator.
+ Pair<ILogicalOperator, LogicalVariable> gbyOpAndAggVar =
+ wrapLimitOrRunningAggregateInGroupBy(op, recordVar, inputLiveVars);
ILogicalOperator gbyOp = gbyOpAndAggVar.first;
LogicalVariable aggVar = gbyOpAndAggVar.second;
gbyOp.getInputs().add(new MutableObject<>(assignOp));
@@ -312,8 +330,8 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
return new Pair<>(assignOp, recordVar);
}
- private Pair<ILogicalOperator, LogicalVariable> wrapLimitInGroupBy(ILogicalOperator op, LogicalVariable recordVar,
- Set<LogicalVariable> inputLiveVars) throws AlgebricksException {
+ private Pair<ILogicalOperator, LogicalVariable> wrapLimitOrRunningAggregateInGroupBy(ILogicalOperator op,
+ LogicalVariable recordVar, Set<LogicalVariable> inputLiveVars) throws AlgebricksException {
SourceLocation sourceLoc = op.getSourceLocation();
GroupByOperator gbyOp = new GroupByOperator();
gbyOp.setSourceLocation(sourceLoc);
@@ -348,7 +366,7 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
AggregateOperator aggOp = new AggregateOperator(aggVarList, aggExprList);
aggOp.setSourceLocation(sourceLoc);
- // Adds the original limit operator as the input operator to the added
+ // Adds the original limit (or running aggregate) operator as the input operator to the added
// aggregate operator.
aggOp.getInputs().add(new MutableObject<>(op));
op.getInputs().clear();
@@ -361,7 +379,7 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
// Adds a nested tuple source operator as the input operator to the
- // limit operator.
+ // limit (or running aggregate) operator.
NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<>(gbyOp));
nts.setSourceLocation(sourceLoc);
currentOp.getInputs().add(new MutableObject<>(nts));
@@ -450,10 +468,11 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
// Updates the primary key info in the copied plan segment.
Map<LogicalVariable, LogicalVariable> varMap = deepCopyVisitor.getInputToOutputVariableMapping();
addPrimaryKeys(varMap);
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(copiedInputOperator, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(copiedInputOperator, true, context);
correlatedKeyVars.clear();
- correlatedKeyVars.addAll(primaryOpAndVars.second);
+ correlatedKeyVars.addAll(primaryOpAndVars.first);
// Update key variables and input-output-var mapping.
varMap.forEach((oldVar, newVar) -> {
if (correlatedKeyVars.contains(oldVar)) {
@@ -462,7 +481,14 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
updateInputToOutputVarMapping(oldVar, newVar, true);
});
- return primaryOpAndVars.first;
+ if (primaryOpAndVars.second == null) {
+ // found primary key for the existing operator (copiedInputOperator)
+ return copiedInputOperator;
+ } else {
+ ILogicalOperator newInputOp = primaryOpAndVars.second;
+ context.addPrimaryKey(primaryOpAndVars.third);
+ return newInputOp;
+ }
}
@Override
@@ -669,19 +695,20 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
/**
- * Wraps an AggregateOperator or RunningAggregateOperator with a group-by
+ * Wraps an AggregateOperator with a group-by
* operator where the group-by keys are variables in keyVarsToEnforce. Note
* that the function here prevents this visitor being used to rewrite
* arbitrary query plans. Instead, it could only be used for rewriting a
* nested plan within a subplan operator.
*
* @param op
- * the logical operator for aggregate or running aggregate.
+ * the logical operator for aggregate.
* @return the wrapped group-by operator if {@code keyVarsToEnforce} is not
* empty, and {@code op} otherwise.
* @throws AlgebricksException
*/
- private ILogicalOperator visitAggregateOperator(ILogicalOperator op) throws AlgebricksException {
+ @Override
+ public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
visitSingleInputOperator(op);
if (correlatedKeyVars.isEmpty()) {
return op;
@@ -808,5 +835,4 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
new FunctionalDependency(newDependencies, Collections.singletonList(entry.getValue())));
}
}
-
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
index 81b4a3d..ae9a886 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
@@ -33,6 +35,7 @@ import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -54,6 +57,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperat
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -365,27 +369,35 @@ public class InlineSubplanInputForNestedTupleSourceRule implements IAlgebraicRew
Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Creates parameters for the left outer join operator.
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(inputOpBackup, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
- ILogicalOperator inputOp = primaryOpAndVars.first;
- Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
- inputOpRef.setValue(inputOp);
+ Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.first;
+ FunctionalDependency newPrimaryKeyFd = null;
+ ILogicalOperator inputOp;
+ if (primaryOpAndVars.second != null) {
+ // there's a new operator that generates a primary key
+ inputOp = primaryOpAndVars.second;
+ inputOpRef.setValue(inputOp);
+ newPrimaryKeyFd = primaryOpAndVars.third;
+ } else {
+ inputOp = inputOpBackup;
+ }
Set<LogicalVariable> inputLiveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs =
- SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
+ SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context, newPrimaryKeyFd);
Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
if (varMap == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, new LinkedHashMap<>());
}
-
Mutable<ILogicalOperator> lowestAggregateRefInSubplan =
SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
if (lowestAggregateRefInSubplan == null) {
- inputOpRef.setValue(inputOpBackup);
- return new Pair<>(false, new LinkedHashMap<>());
+ // not supposed to happen. SubplanFlatteningUtil.inlineAllNestedTupleSource() checks for it
+ throw new CompilationException(ErrorCode.ILLEGAL_STATE, subplanOp.getSourceLocation());
}
Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
@@ -525,21 +537,29 @@ public class InlineSubplanInputForNestedTupleSourceRule implements IAlgebraicRew
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Gets live variables and covering variables from the subplan's input operator.
- Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars =
+ EquivalenceClassUtils.computePrimaryKeys(inputOpBackup, context);
+ Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> primaryOpAndVars =
EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, false, context);
- ILogicalOperator inputOp = primaryOpAndVars.first;
- Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
- inputOpRef.setValue(inputOp);
+ Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.first;
+ FunctionalDependency newPrimaryKeyFd = null;
+ ILogicalOperator inputOp;
+ if (primaryOpAndVars.second != null) {
+ // there's a new operator that generates a primary key
+ inputOp = primaryOpAndVars.second;
+ inputOpRef.setValue(inputOp);
+ newPrimaryKeyFd = primaryOpAndVars.third;
+ } else {
+ inputOp = inputOpBackup;
+ }
Set<LogicalVariable> liveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, liveVars);
Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> notNullVarsAndTopJoinRef =
- SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context);
+ SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context, newPrimaryKeyFd);
if (notNullVarsAndTopJoinRef.first == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, replacedVarMap);
}
-
Set<LogicalVariable> notNullVars = notNullVarsAndTopJoinRef.first;
Mutable<ILogicalOperator> topJoinRef = notNullVarsAndTopJoinRef.second;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
index 09377fa..e3c7d69 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanFlatteningUtil.java
@@ -35,28 +35,35 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import com.google.common.collect.ImmutableSet;
-
class SubplanFlatteningUtil {
+ private static final Set<LogicalOperatorTag> OP_SET_NESTEDTUPLESOURCE =
+ EnumSet.of(LogicalOperatorTag.NESTEDTUPLESOURCE);
+
+ private static final Set<LogicalOperatorTag> OP_SET_INNER_OUTER_JOIN =
+ EnumSet.of(LogicalOperatorTag.INNERJOIN, LogicalOperatorTag.LEFTOUTERJOIN);
+
/**
* Blindly inline all NTS's in a Subplan operator.
*
* @param subplanOp,
* the subplan operator
* @param context
+ * the optimization context
+ * @param extraPrimaryKeyFd
+ * extra primary key dependency that needs to be added to the context before performing the rewrite
* @return a map that maps primary key variables in the subplan's input to its deep copies
* in the nested pipeline; the ordering that needs to be maintained for the final
* aggregation in the added group-by operator.
- * @throws AlgebricksException
*/
public static Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> inlineAllNestedTupleSource(
- SubplanOperator subplanOp, IOptimizationContext context) throws AlgebricksException {
+ SubplanOperator subplanOp, IOptimizationContext context, FunctionalDependency extraPrimaryKeyFd)
+ throws AlgebricksException {
// For nested subplan, we do not continue for the general inlining.
- if (OperatorManipulationUtil.ancestorOfOperators(subplanOp,
- ImmutableSet.of(LogicalOperatorTag.NESTEDTUPLESOURCE))) {
+ if (OperatorManipulationUtil.ancestorOfOperators(subplanOp, OP_SET_NESTEDTUPLESOURCE)) {
return new Pair<>(null, null);
}
@@ -65,6 +72,10 @@ class SubplanFlatteningUtil {
return new Pair<>(null, null);
}
+ if (extraPrimaryKeyFd != null) {
+ context.addPrimaryKey(extraPrimaryKeyFd);
+ }
+
// Rewrites the query plan.
InlineAllNtsInSubplanVisitor visitor = new InlineAllNtsInSubplanVisitor(context, subplanOp);
ILogicalOperator topOp = topOpRef.getValue();
@@ -77,8 +88,7 @@ class SubplanFlatteningUtil {
// Gets ordering variables.
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderVars = visitor.getOrderingExpressions();
- return new Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>>(
- visitor.getInputVariableToOutputVariableMap(), orderVars);
+ return new Pair<>(visitor.getInputVariableToOutputVariableMap(), orderVars);
}
/**
@@ -89,17 +99,23 @@ class SubplanFlatteningUtil {
* the SubplanOperator
* @param context
* the optimization context
+ * @param extraPrimaryKeyFd
+ * extra primary key dependency that needs to be added to the context before performing the rewrite
* @return A set of variables used for further null-checks, i.e., variables indicating
* whether a tuple produced by a transformed left outer join is a non-match;
* a reference to the top join operator in the nested subplan.
- * @throws AlgebricksException
*/
public static Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> inlineLeftNtsInSubplanJoin(
- SubplanOperator subplanOp, IOptimizationContext context) throws AlgebricksException {
+ SubplanOperator subplanOp, IOptimizationContext context, FunctionalDependency extraPrimaryKeyFd)
+ throws AlgebricksException {
Pair<Boolean, ILogicalOperator> applicableAndNtsToRewrite =
SubplanFlatteningUtil.isQualifiedForSpecialFlattening(subplanOp);
if (!applicableAndNtsToRewrite.first) {
- return new Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>>(null, null);
+ return new Pair<>(null, null);
+ }
+
+ if (extraPrimaryKeyFd != null) {
+ context.addPrimaryKey(extraPrimaryKeyFd);
}
ILogicalOperator qualifiedNts = applicableAndNtsToRewrite.second;
@@ -126,8 +142,7 @@ class SubplanFlatteningUtil {
VariableUtilities.substituteVariables(currentOp, subplanLocalVarMap, context);
currentOp = currentOp.getInputs().get(0).getValue();
}
- return new Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>>(specialVisitor.getNullCheckVariables(),
- topJoinRef);
+ return new Pair<>(specialVisitor.getNullCheckVariables(), topJoinRef);
}
/**
@@ -206,25 +221,23 @@ class SubplanFlatteningUtil {
* @param subplanOp,
* the SubplanOperator to consider
* @return TRUE if the rewriting is applicable; FALSE otherwise.
- * @throws AlgebricksException
*/
private static Pair<Boolean, ILogicalOperator> isQualifiedForSpecialFlattening(SubplanOperator subplanOp)
throws AlgebricksException {
if (!OperatorManipulationUtil.ancestorOfOperators(
subplanOp.getNestedPlans().get(0).getRoots().get(0).getValue(),
// we don't need to check recursively for this special rewriting.
- EnumSet.of(LogicalOperatorTag.INNERJOIN, LogicalOperatorTag.LEFTOUTERJOIN))) {
- return new Pair<Boolean, ILogicalOperator>(false, null);
+ OP_SET_INNER_OUTER_JOIN)) {
+ return new Pair<>(false, null);
}
SubplanSpecialFlatteningCheckVisitor visitor = new SubplanSpecialFlatteningCheckVisitor();
for (ILogicalPlan plan : subplanOp.getNestedPlans()) {
for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
if (!opRef.getValue().accept(visitor, null)) {
- return new Pair<Boolean, ILogicalOperator>(false, null);
+ return new Pair<>(false, null);
}
}
}
- return new Pair<Boolean, ILogicalOperator>(true, visitor.getQualifiedNts());
+ return new Pair<>(true, visitor.getQualifiedNts());
}
-
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index 1593cf1..24f368f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -22,40 +22,49 @@ package org.apache.asterix.optimizer.rules.util;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.PrimaryKeyVariablesVisitor;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
public class EquivalenceClassUtils {
+ // Controls whether to use create-query-uid() function when generating a primary key
+ // If disabled then use row_number() approach instead
+ public static final String REWRITE_INTERNAL_QUERYUID_PK = "rewrite_internal_queryuid_pk";
+ static final boolean REWRITE_INTERNAL_QUERYUID_PK_DEFAULT = true;
+
/**
* Adds equivalent classes for primary index accesses, including unnest-map for
* primary index access and data source scan through primary index ---
@@ -148,16 +157,14 @@ public class EquivalenceClassUtils {
* a set of primary key variables at the operator.
* @throws AlgebricksException
*/
- public static Pair<ILogicalOperator, Set<LogicalVariable>> findOrCreatePrimaryKeyOpAndVariables(
+ public static Triple<Set<LogicalVariable>, ILogicalOperator, FunctionalDependency> findOrCreatePrimaryKeyOpAndVariables(
ILogicalOperator operator, boolean usedForCorrelationJoin, IOptimizationContext context)
throws AlgebricksException {
- computePrimaryKeys(operator, context);
-
- Set<LogicalVariable> liveVars = new HashSet<>();
+ Set<LogicalVariable> liveVars = new LinkedHashSet<>();
VariableUtilities.getSubplanLocalLiveVariables(operator, liveVars);
- Set<LogicalVariable> primaryKeyVars = new HashSet<>();
- Set<LogicalVariable> noKeyVars = new HashSet<>();
+ Set<LogicalVariable> primaryKeyVars = new LinkedHashSet<>();
+ Set<LogicalVariable> noKeyVars = new LinkedHashSet<>();
for (LogicalVariable liveVar : liveVars) {
List<LogicalVariable> keyVars = context.findPrimaryKey(liveVar);
if (keyVars != null) {
@@ -171,24 +178,57 @@ public class EquivalenceClassUtils {
}
primaryKeyVars.retainAll(liveVars);
if (primaryKeyVars.containsAll(noKeyVars)) {
- return new Pair<ILogicalOperator, Set<LogicalVariable>>(operator, primaryKeyVars);
+ return new Triple<>(primaryKeyVars, null, null);
+ } else if (!usedForCorrelationJoin && isQueryUidPkEnabled(context)) {
+ LogicalVariable idVar = context.newVar();
+ RunningAggregateOperator assignIdOp =
+ new RunningAggregateOperator(idVar, new MutableObject<>(new StatefulFunctionCallExpression(
+ BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.CREATE_QUERY_UID), null)));
+ assignIdOp.setSourceLocation(operator.getSourceLocation());
+ assignIdOp.getInputs().add(new MutableObject<>(operator));
+
+ context.computeAndSetTypeEnvironmentForOperator(assignIdOp);
+
+ FunctionalDependency primaryKeyFD =
+ new FunctionalDependency(Collections.singletonList(idVar), new ArrayList<>(liveVars));
+
+ return new Triple<>(Collections.singleton(idVar), assignIdOp, primaryKeyFD);
} else {
- LogicalVariable assignVar = context.newVar();
- ILogicalOperator assignOp = new AssignOperator(assignVar,
- new MutableObject<ILogicalExpression>(new StatefulFunctionCallExpression(
- FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_QUERY_UID), null)));
- OperatorPropertiesUtil.markMovable(assignOp, !usedForCorrelationJoin);
- assignOp.getInputs().add(new MutableObject<ILogicalOperator>(operator));
- context.addPrimaryKey(new FunctionalDependency(Collections.singletonList(assignVar),
- new ArrayList<LogicalVariable>(liveVars)));
- context.computeAndSetTypeEnvironmentForOperator(assignOp);
- return new Pair<ILogicalOperator, Set<LogicalVariable>>(assignOp, Collections.singleton(assignVar));
+ noKeyVars.removeAll(primaryKeyVars);
+
+ List<Mutable<ILogicalExpression>> partitionVarRefs =
+ new ArrayList<>(primaryKeyVars.size() + noKeyVars.size());
+ OperatorManipulationUtil.createVariableReferences(primaryKeyVars, operator.getSourceLocation(),
+ partitionVarRefs);
+ OperatorManipulationUtil.createVariableReferences(noKeyVars, operator.getSourceLocation(),
+ partitionVarRefs);
+ LogicalVariable rowNumVar = context.newVar();
+ AbstractFunctionCallExpression rowNumExpr = BuiltinFunctions
+ .makeWindowFunctionExpression(BuiltinFunctions.ROW_NUMBER_IMPL, Collections.emptyList());
+ WindowOperator winOp = new WindowOperator(partitionVarRefs, Collections.emptyList());
+ winOp.getVariables().add(rowNumVar);
+ winOp.getExpressions().add(new MutableObject<>(rowNumExpr));
+ winOp.setSourceLocation(operator.getSourceLocation());
+ winOp.getInputs().add(new MutableObject<>(operator));
+
+ context.computeAndSetTypeEnvironmentForOperator(winOp);
+
+ primaryKeyVars.addAll(noKeyVars);
+ primaryKeyVars.add(rowNumVar);
+ FunctionalDependency primaryKeyFD =
+ new FunctionalDependency(new ArrayList<>(primaryKeyVars), new ArrayList<>(liveVars));
+
+ return new Triple<>(primaryKeyVars, winOp, primaryKeyFD);
}
}
- private static void computePrimaryKeys(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ public static void computePrimaryKeys(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
PrimaryKeyVariablesVisitor visitor = new PrimaryKeyVariablesVisitor();
PhysicalOptimizationsUtil.visitOperatorAndItsDescendants(op, visitor, ctx);
}
+ private static boolean isQueryUidPkEnabled(IOptimizationContext context) {
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ return metadataProvider.getBooleanProperty(REWRITE_INTERNAL_QUERYUID_PK, REWRITE_INTERNAL_QUERYUID_PK_DEFAULT);
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index d60047e..0091304 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -73,6 +73,7 @@ import org.apache.asterix.optimizer.base.AsterixOptimizationContext;
import org.apache.asterix.optimizer.base.FuzzyUtils;
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule;
+import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.ExecutionPlans;
@@ -143,7 +144,8 @@ public class APIFramework {
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
- SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN);
+ SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
+ EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
similarity index 62%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
index 056d96e..3699c56 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/in_let_8.sqlpp
@@ -17,8 +17,29 @@
* under the License.
*/
-use tpch;
+drop dataverse test if exists;
+create dataverse test;
-select distinct value print_binary(id, "hex") from LineItem
-let id = create_query_uid()
-order by id;
\ No newline at end of file
+use test;
+
+SET `compiler.sort.parallel` "false";
+SET `rewrite_internal_queryuid_pk` "false";
+
+create type test.TestType as
+{
+ id : integer
+};
+
+create dataset cart(TestType) primary key id;
+
+select c1.cid, i1.pid, i1.ts
+from cart c1 unnest c1.items i1
+where i1.ts >= 2000 and i1.pid in
+(
+ select value i2.pid
+ from cart c2 unnest c2.items i2
+ where i2.ts >= 2000
+ group by i2.pid
+ having count(*) > 1
+)
+order by c1.cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
index 4494cfa..4299ab7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2845.sqlpp
@@ -35,7 +35,7 @@ WITH
j AS (
SELECT jid, a
FROM jds
-), ---> 3 rows (jid=1, 2, 3)
+),
m1 AS (
SELECT jid, x, COUNT(1) c1
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
index f6599e6..da74507 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-2402.plan
@@ -50,7 +50,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$309][$$222] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$309] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- UNNEST |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -93,12 +93,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
@@ -139,12 +140,13 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (channels.EmergenciesNearMeChannelChannelSubscriptions) |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
index 6d569dc..646ab5b 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
@@ -10,54 +10,87 @@
-- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
- -- HASH_PARTITION_MERGE_EXCHANGE MERGE:[$$taskId(ASC)] HASH:[$$taskId] |PARTITIONED|
- -- SORT_GROUP_BY[$$279] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$279] |PARTITIONED|
- -- SORT_GROUP_BY[$$242] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
+ -- STREAM_PROJECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
+ -- HASH_PARTITION_MERGE_EXCHANGE MERGE:[$$taskId(ASC)] HASH:[$$taskId] |PARTITIONED|
+ -- SORT_GROUP_BY[$$279] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$279] |PARTITIONED|
+ -- SORT_GROUP_BY[$$242] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
+ -- SORT_GROUP_BY[$$281] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$281] |PARTITIONED|
+ -- SORT_GROUP_BY[$$243] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$281] |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$283] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$283] |PARTITIONED|
+ -- SORT_GROUP_BY[$$244] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$281] |PARTITIONED|
- -- SORT_GROUP_BY[$$243] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -67,35 +100,6 @@
-- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$283] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$283] |PARTITIONED|
- -- SORT_GROUP_BY[$$244] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
index 29fcd1d..8230bf0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-issue562_ps.plan
@@ -46,20 +46,21 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$146][$$153] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$153] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
@@ -112,20 +113,21 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$146][$$153] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- BROADCAST_EXCHANGE |PARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$phone_substr][$$148] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpch.Customer) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$153] |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
index 5d16539..fd70464 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
@@ -22,9 +22,11 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$45][$$44] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$45] |PARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- STREAM_PROJECT |UNPARTITIONED|
+ -- RUNNING_AGGREGATE |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
index cb471a9..b81c831 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists.plan
@@ -40,7 +40,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
index 52d6e25..2cccba4 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/exists_ps.plan
@@ -46,7 +46,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
@@ -123,7 +123,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$171] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
index 3464eae..ec77d62 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$97][$$pid] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
index a500b92..ee9243d 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_4.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$116][$$108] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$116] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
index 88cacfa..c670194 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_5.plan
@@ -22,8 +22,8 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$116][$$108] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
copy to asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan
index 3464eae..878c508 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_3.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/in_let_8.plan
@@ -9,48 +9,47 @@
-- STREAM_SELECT |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$104] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$94, $$i1, $$97, $$93, $$104] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- STREAM_SELECT |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$104(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$104] |PARTITIONED|
+ -- STABLE_SORT [$$94(ASC), $$i1(ASC), $$97(ASC), $$93(ASC), $$104(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$97][$$pid] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- UNNEST |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$94(ASC), $$i1(ASC), $$97(ASC), $$93(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- STREAM_SELECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$108] |PARTITIONED|
+ -- SORT_GROUP_BY[$$112] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$108] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$112] |PARTITIONED|
-- SORT_GROUP_BY[$$92] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -64,11 +63,10 @@
-- UNNEST |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.cart) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
index d631085..5501c67 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists.plan
@@ -40,7 +40,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
index 5ab7d30..c8944e7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/not_exists_ps.plan
@@ -46,7 +46,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
@@ -123,7 +123,7 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$175][$$172] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$175] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- NESTED_LOOP |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
index 7cbb67f..4b32064 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2845.plan
@@ -7,75 +7,38 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$295][$$296] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$295] |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$277][$$jid] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$jid] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$319, $$320] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$319, $$320] |PARTITIONED|
- -- SORT_GROUP_BY[$$273, $$274] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$296] |PARTITIONED|
- -- NESTED_LOOP |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$277, $$240, $$290, $$287, $#4, $$295][$$303, $$308, $$298, $$299, $$300, $$301] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$303][$$306] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- STABLE_SORT [$$277(ASC), $$240(ASC), $$290(ASC), $$287(ASC), $#4(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$277][$$jid] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$306] |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$jid] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$322, $$323] |PARTITIONED|
+ -- SORT_GROUP_BY[$$324, $$325] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$322, $$323] |PARTITIONED|
- -- SORT_GROUP_BY[$$311, $$312] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$324, $$325] |PARTITIONED|
+ -- SORT_GROUP_BY[$$273, $$274] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
@@ -83,26 +46,74 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$303] |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$303(ASC), $$308(ASC), $$298(ASC), $$299(ASC), $$300(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$303][$$306] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.jds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$306] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$327, $$328] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$327, $$328] |PARTITIONED|
+ -- SORT_GROUP_BY[$$311, $$312] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.mds) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- BROADCAST_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$325, $$326] |PARTITIONED|
+ -- SORT_GROUP_BY[$$330, $$331] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$325, $$326] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$330, $$331] |PARTITIONED|
-- SORT_GROUP_BY[$$275, $$276] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
index 972e59e..bda21dc 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated-2.plan
@@ -55,26 +55,26 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$187] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$234] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$234] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$186] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$186(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$233, $$234] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$233, $$234] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$181, $$186] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$181(ASC), $$186(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$170][$$169] |PARTITIONED|
@@ -84,49 +84,50 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$169] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -138,171 +139,173 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$225] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$240] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$240] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$224] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$224(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$239, $$240] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$239, $$240] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$191, $$224] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$191(ASC), $$224(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$174][$$173] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$174] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$194][$$171] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$195] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$194][$$171] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$195] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$195][$$197] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$195][$$197] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$197] |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$237] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$237] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$203] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$236, $$237] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$203(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$236, $$237] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$202, $$203] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$204][$$207] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- STABLE_SORT [$$202(ASC), $$203(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$204][$$207] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$204] |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$208] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$208][$$209] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$207] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$173] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
index 1a70178..d61947a 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/tpcds/query-ASTERIXDB-1581-correlated.plan
@@ -73,26 +73,26 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$171] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$221] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$221] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$161] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$161(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$220, $$221] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$220, $$221] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$158, $$161] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$158(ASC), $$161(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$150][$$149] |PARTITIONED|
@@ -102,49 +102,50 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
@@ -156,175 +157,177 @@
-- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$215] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$227] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$227] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$177] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$177(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$226, $$227] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$226, $$227] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$174, $$177] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$174(ASC), $$177(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$152][$$151] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$152] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$178] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$178] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$178(ASC)] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$178][$$181] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$178(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$178][$$181] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$224] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$224] |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$194] |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$181] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$223, $$224] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$194(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$223, $$224] |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$193, $$194] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$196][$$198] |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- STABLE_SORT [$$193(ASC), $$194(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$196][$$198] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$196] |PARTITIONED|
+ -- RUNNING_AGGREGATE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- AGGREGATE |LOCAL|
- -- STREAM_SELECT |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$199] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$199][$$200] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- BTREE_SEARCH (tpcds.item.item) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$198] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (tpcds.store_sales) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$151] |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp
deleted file mode 100644
index d3ba2fb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.1.ddl.sqlpp
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-drop dataverse tpch if exists;
-create dataverse tpch;
-
-use tpch;
-
-create type LineItemType as closed {
- l_orderkey: int64,
- l_partkey: int64,
- l_suppkey: int64,
- l_linenumber: int64,
- l_quantity: double,
- l_extendedprice: double,
- l_discount: double,
- l_tax: double,
- l_returnflag: string,
- l_linestatus: string,
- l_shipdate: string,
- l_commitdate: string,
- l_receiptdate: string,
- l_shipinstruct: string,
- l_shipmode: string,
- l_comment: string
-};
-
-create dataset LineItem(LineItemType)
- primary key l_orderkey, l_linenumber;
-
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
similarity index 67%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
index 5812379..77f6027 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/binary/query_id/query_id.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/in_let/in_let.8.query.sqlpp
@@ -17,8 +17,23 @@
* under the License.
*/
-use tpch;
+/*
+ * Test "rewrite_internal_queryuid_pk" option (default is true)
+ */
+
+use test;
+
+SET `compiler.sort.parallel` "false";
+SET `rewrite_internal_queryuid_pk` "false";
-load dataset LineItem
-using localfs
-(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+select c1.cid, i1.pid, i1.ts
+from cart c1 unnest c1.items i1
+where i1.ts >= 2000 and i1.pid in
+(
+ select value i2.pid
+ from cart c2 unnest c2.items i2
+ where i2.ts >= 2000
+ group by i2.pid
+ having count(*) > 1
+)
+order by c1.cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm
deleted file mode 100644
index dcfde8d..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/binary/query_id/query_id.1.adm
+++ /dev/null
@@ -1,6005 +0,0 @@
-"000000000000000000000001"
-"000000000000000000000002"
-"000000000000000000000003"
-"000000000000000000000004"
-"000000000000000000000005"
-"000000000000000000000006"
-"000000000000000000000007"
-"000000000000000000000008"
-"000000000000000000000009"
-"00000000000000000000000A"
-"00000000000000000000000B"
-"00000000000000000000000C"
-"00000000000000000000000D"
-"00000000000000000000000E"
-"00000000000000000000000F"
-"000000000000000000000010"
-"000000000000000000000011"
-"000000000000000000000012"
-"000000000000000000000013"
-"000000000000000000000014"
-"000000000000000000000015"
-"000000000000000000000016"
-"000000000000000000000017"
-"000000000000000000000018"
-"000000000000000000000019"
-"00000000000000000000001A"
-"00000000000000000000001B"
-"00000000000000000000001C"
-"00000000000000000000001D"
-"00000000000000000000001E"
-"00000000000000000000001F"
-"000000000000000000000020"
-"000000000000000000000021"
-"000000000000000000000022"
-"000000000000000000000023"
-"000000000000000000000024"
-"000000000000000000000025"
-"000000000000000000000026"
-"000000000000000000000027"
-"000000000000000000000028"
-"000000000000000000000029"
-"00000000000000000000002A"
-"00000000000000000000002B"
-"00000000000000000000002C"
-"00000000000000000000002D"
-"00000000000000000000002E"
-"00000000000000000000002F"
-"000000000000000000000030"
-"000000000000000000000031"
-"000000000000000000000032"
-"000000000000000000000033"
-"000000000000000000000034"
-"000000000000000000000035"
-"000000000000000000000036"
-"000000000000000000000037"
-"000000000000000000000038"
-"000000000000000000000039"
-"00000000000000000000003A"
-"00000000000000000000003B"
-"00000000000000000000003C"
-"00000000000000000000003D"
-"00000000000000000000003E"
-"00000000000000000000003F"
-"000000000000000000000040"
-"000000000000000000000041"
-"000000000000000000000042"
-"000000000000000000000043"
-"000000000000000000000044"
-"000000000000000000000045"
-"000000000000000000000046"
-"000000000000000000000047"
-"000000000000000000000048"
-"000000000000000000000049"
-"00000000000000000000004A"
-"00000000000000000000004B"
-"00000000000000000000004C"
-"00000000000000000000004D"
-"00000000000000000000004E"
-"00000000000000000000004F"
-"000000000000000000000050"
-"000000000000000000000051"
-"000000000000000000000052"
-"000000000000000000000053"
-"000000000000000000000054"
-"000000000000000000000055"
-"000000000000000000000056"
-"000000000000000000000057"
-"000000000000000000000058"
-"000000000000000000000059"
-"00000000000000000000005A"
-"00000000000000000000005B"
-"00000000000000000000005C"
-"00000000000000000000005D"
-"00000000000000000000005E"
-"00000000000000000000005F"
-"000000000000000000000060"
-"000000000000000000000061"
-"000000000000000000000062"
-"000000000000000000000063"
-"000000000000000000000064"
-"000000000000000000000065"
-"000000000000000000000066"
-"000000000000000000000067"
-"000000000000000000000068"
-"000000000000000000000069"
-"00000000000000000000006A"
-"00000000000000000000006B"
-"00000000000000000000006C"
-"00000000000000000000006D"
-"00000000000000000000006E"
-"00000000000000000000006F"
-"000000000000000000000070"
-"000000000000000000000071"
-"000000000000000000000072"
-"000000000000000000000073"
-"000000000000000000000074"
-"000000000000000000000075"
-"000000000000000000000076"
-"000000000000000000000077"
-"000000000000000000000078"
-"000000000000000000000079"
-"00000000000000000000007A"
-"00000000000000000000007B"
-"00000000000000000000007C"
-"00000000000000000000007D"
-"00000000000000000000007E"
-"00000000000000000000007F"
-"000000000000000000000080"
-"000000000000000000000081"
-"000000000000000000000082"
-"000000000000000000000083"
-"000000000000000000000084"
-"000000000000000000000085"
-"000000000000000000000086"
-"000000000000000000000087"
-"000000000000000000000088"
-"000000000000000000000089"
-"00000000000000000000008A"
-"00000000000000000000008B"
-"00000000000000000000008C"
-"00000000000000000000008D"
-"00000000000000000000008E"
-"00000000000000000000008F"
-"000000000000000000000090"
-"000000000000000000000091"
-"000000000000000000000092"
-"000000000000000000000093"
-"000000000000000000000094"
-"000000000000000000000095"
-"000000000000000000000096"
-"000000000000000000000097"
-"000000000000000000000098"
-"000000000000000000000099"
-"00000000000000000000009A"
-"00000000000000000000009B"
-"00000000000000000000009C"
-"00000000000000000000009D"
-"00000000000000000000009E"
-"00000000000000000000009F"
-"0000000000000000000000A0"
-"0000000000000000000000A1"
-"0000000000000000000000A2"
-"0000000000000000000000A3"
-"0000000000000000000000A4"
-"0000000000000000000000A5"
-"0000000000000000000000A6"
-"0000000000000000000000A7"
-"0000000000000000000000A8"
-"0000000000000000000000A9"
-"0000000000000000000000AA"
-"0000000000000000000000AB"
-"0000000000000000000000AC"
-"0000000000000000000000AD"
-"0000000000000000000000AE"
-"0000000000000000000000AF"
-"0000000000000000000000B0"
-"0000000000000000000000B1"
-"0000000000000000000000B2"
-"0000000000000000000000B3"
-"0000000000000000000000B4"
-"0000000000000000000000B5"
-"0000000000000000000000B6"
-"0000000000000000000000B7"
-"0000000000000000000000B8"
-"0000000000000000000000B9"
-"0000000000000000000000BA"
-"0000000000000000000000BB"
-"0000000000000000000000BC"
-"0000000000000000000000BD"
-"0000000000000000000000BE"
-"0000000000000000000000BF"
-"0000000000000000000000C0"
-"0000000000000000000000C1"
-"0000000000000000000000C2"
-"0000000000000000000000C3"
-"0000000000000000000000C4"
-"0000000000000000000000C5"
-"0000000000000000000000C6"
-"0000000000000000000000C7"
-"0000000000000000000000C8"
-"0000000000000000000000C9"
-"0000000000000000000000CA"
-"0000000000000000000000CB"
-"0000000000000000000000CC"
-"0000000000000000000000CD"
-"0000000000000000000000CE"
-"0000000000000000000000CF"
-"0000000000000000000000D0"
-"0000000000000000000000D1"
-"0000000000000000000000D2"
-"0000000000000000000000D3"
-"0000000000000000000000D4"
-"0000000000000000000000D5"
-"0000000000000000000000D6"
-"0000000000000000000000D7"
-"0000000000000000000000D8"
-"0000000000000000000000D9"
-"0000000000000000000000DA"
-"0000000000000000000000DB"
-"0000000000000000000000DC"
-"0000000000000000000000DD"
-"0000000000000000000000DE"
-"0000000000000000000000DF"
-"0000000000000000000000E0"
-"0000000000000000000000E1"
-"0000000000000000000000E2"
-"0000000000000000000000E3"
-"0000000000000000000000E4"
-"0000000000000000000000E5"
-"0000000000000000000000E6"
-"0000000000000000000000E7"
-"0000000000000000000000E8"
-"0000000000000000000000E9"
-"0000000000000000000000EA"
-"0000000000000000000000EB"
-"0000000000000000000000EC"
-"0000000000000000000000ED"
-"0000000000000000000000EE"
-"0000000000000000000000EF"
-"0000000000000000000000F0"
-"0000000000000000000000F1"
-"0000000000000000000000F2"
-"0000000000000000000000F3"
-"0000000000000000000000F4"
-"0000000000000000000000F5"
-"0000000000000000000000F6"
-"0000000000000000000000F7"
-"0000000000000000000000F8"
-"0000000000000000000000F9"
-"0000000000000000000000FA"
-"0000000000000000000000FB"
-"0000000000000000000000FC"
-"0000000000000000000000FD"
-"0000000000000000000000FE"
-"0000000000000000000000FF"
-"000000000000000000000100"
-"000000000000000000000101"
-"000000000000000000000102"
-"000000000000000000000103"
-"000000000000000000000104"
-"000000000000000000000105"
-"000000000000000000000106"
-"000000000000000000000107"
-"000000000000000000000108"
-"000000000000000000000109"
-"00000000000000000000010A"
-"00000000000000000000010B"
-"00000000000000000000010C"
-"00000000000000000000010D"
-"00000000000000000000010E"
-"00000000000000000000010F"
-"000000000000000000000110"
-"000000000000000000000111"
-"000000000000000000000112"
-"000000000000000000000113"
-"000000000000000000000114"
-"000000000000000000000115"
-"000000000000000000000116"
-"000000000000000000000117"
-"000000000000000000000118"
-"000000000000000000000119"
-"00000000000000000000011A"
-"00000000000000000000011B"
-"00000000000000000000011C"
-"00000000000000000000011D"
-"00000000000000000000011E"
-"00000000000000000000011F"
-"000000000000000000000120"
-"000000000000000000000121"
-"000000000000000000000122"
-"000000000000000000000123"
-"000000000000000000000124"
-"000000000000000000000125"
-"000000000000000000000126"
-"000000000000000000000127"
-"000000000000000000000128"
-"000000000000000000000129"
-"00000000000000000000012A"
-"00000000000000000000012B"
-"00000000000000000000012C"
-"00000000000000000000012D"
-"00000000000000000000012E"
-"00000000000000000000012F"
-"000000000000000000000130"
-"000000000000000000000131"
-"000000000000000000000132"
-"000000000000000000000133"
-"000000000000000000000134"
-"000000000000000000000135"
-"000000000000000000000136"
-"000000000000000000000137"
-"000000000000000000000138"
-"000000000000000000000139"
-"00000000000000000000013A"
-"00000000000000000000013B"
-"00000000000000000000013C"
-"00000000000000000000013D"
-"00000000000000000000013E"
-"00000000000000000000013F"
-"000000000000000000000140"
-"000000000000000000000141"
-"000000000000000000000142"
-"000000000000000000000143"
-"000000000000000000000144"
-"000000000000000000000145"
-"000000000000000000000146"
-"000000000000000000000147"
-"000000000000000000000148"
-"000000000000000000000149"
-"00000000000000000000014A"
-"00000000000000000000014B"
-"00000000000000000000014C"
-"00000000000000000000014D"
-"00000000000000000000014E"
-"00000000000000000000014F"
-"000000000000000000000150"
-"000000000000000000000151"
-"000000000000000000000152"
-"000000000000000000000153"
-"000000000000000000000154"
-"000000000000000000000155"
-"000000000000000000000156"
-"000000000000000000000157"
-"000000000000000000000158"
-"000000000000000000000159"
-"00000000000000000000015A"
-"00000000000000000000015B"
-"00000000000000000000015C"
-"00000000000000000000015D"
-"00000000000000000000015E"
-"00000000000000000000015F"
-"000000000000000000000160"
-"000000000000000000000161"
-"000000000000000000000162"
-"000000000000000000000163"
-"000000000000000000000164"
-"000000000000000000000165"
-"000000000000000000000166"
-"000000000000000000000167"
-"000000000000000000000168"
-"000000000000000000000169"
-"00000000000000000000016A"
-"00000000000000000000016B"
-"00000000000000000000016C"
-"00000000000000000000016D"
-"00000000000000000000016E"
-"00000000000000000000016F"
-"000000000000000000000170"
-"000000000000000000000171"
-"000000000000000000000172"
-"000000000000000000000173"
-"000000000000000000000174"
-"000000000000000000000175"
-"000000000000000000000176"
-"000000000000000000000177"
-"000000000000000000000178"
-"000000000000000000000179"
-"00000000000000000000017A"
-"00000000000000000000017B"
-"00000000000000000000017C"
-"00000000000000000000017D"
-"00000000000000000000017E"
-"00000000000000000000017F"
-"000000000000000000000180"
-"000000000000000000000181"
-"000000000000000000000182"
-"000000000000000000000183"
-"000000000000000000000184"
-"000000000000000000000185"
-"000000000000000000000186"
-"000000000000000000000187"
-"000000000000000000000188"
-"000000000000000000000189"
-"00000000000000000000018A"
-"00000000000000000000018B"
-"00000000000000000000018C"
-"00000000000000000000018D"
-"00000000000000000000018E"
-"00000000000000000000018F"
-"000000000000000000000190"
-"000000000000000000000191"
-"000000000000000000000192"
-"000000000000000000000193"
-"000000000000000000000194"
-"000000000000000000000195"
-"000000000000000000000196"
-"000000000000000000000197"
-"000000000000000000000198"
-"000000000000000000000199"
-"00000000000000000000019A"
-"00000000000000000000019B"
-"00000000000000000000019C"
-"00000000000000000000019D"
-"00000000000000000000019E"
-"00000000000000000000019F"
-"0000000000000000000001A0"
-"0000000000000000000001A1"
-"0000000000000000000001A2"
-"0000000000000000000001A3"
-"0000000000000000000001A4"
-"0000000000000000000001A5"
-"0000000000000000000001A6"
-"0000000000000000000001A7"
-"0000000000000000000001A8"
-"0000000000000000000001A9"
-"0000000000000000000001AA"
-"0000000000000000000001AB"
-"0000000000000000000001AC"
-"0000000000000000000001AD"
-"0000000000000000000001AE"
-"0000000000000000000001AF"
-"0000000000000000000001B0"
-"0000000000000000000001B1"
-"0000000000000000000001B2"
-"0000000000000000000001B3"
-"0000000000000000000001B4"
-"0000000000000000000001B5"
-"0000000000000000000001B6"
-"0000000000000000000001B7"
-"0000000000000000000001B8"
-"0000000000000000000001B9"
-"0000000000000000000001BA"
-"0000000000000000000001BB"
-"0000000000000000000001BC"
-"0000000000000000000001BD"
-"0000000000000000000001BE"
-"0000000000000000000001BF"
-"0000000000000000000001C0"
-"0000000000000000000001C1"
-"0000000000000000000001C2"
-"0000000000000000000001C3"
-"0000000000000000000001C4"
-"0000000000000000000001C5"
-"0000000000000000000001C6"
-"0000000000000000000001C7"
-"0000000000000000000001C8"
-"0000000000000000000001C9"
-"0000000000000000000001CA"
-"0000000000000000000001CB"
-"0000000000000000000001CC"
-"0000000000000000000001CD"
-"0000000000000000000001CE"
-"0000000000000000000001CF"
-"0000000000000000000001D0"
-"0000000000000000000001D1"
-"0000000000000000000001D2"
-"0000000000000000000001D3"
-"0000000000000000000001D4"
-"0000000000000000000001D5"
-"0000000000000000000001D6"
-"0000000000000000000001D7"
-"0000000000000000000001D8"
-"0000000000000000000001D9"
-"0000000000000000000001DA"
-"0000000000000000000001DB"
-"0000000000000000000001DC"
-"0000000000000000000001DD"
-"0000000000000000000001DE"
-"0000000000000000000001DF"
-"0000000000000000000001E0"
-"0000000000000000000001E1"
-"0000000000000000000001E2"
-"0000000000000000000001E3"
-"0000000000000000000001E4"
-"0000000000000000000001E5"
-"0000000000000000000001E6"
-"0000000000000000000001E7"
-"0000000000000000000001E8"
-"0000000000000000000001E9"
-"0000000000000000000001EA"
-"0000000000000000000001EB"
-"0000000000000000000001EC"
-"0000000000000000000001ED"
-"0000000000000000000001EE"
-"0000000000000000000001EF"
-"0000000000000000000001F0"
-"0000000000000000000001F1"
-"0000000000000000000001F2"
-"0000000000000000000001F3"
-"0000000000000000000001F4"
-"0000000000000000000001F5"
-"0000000000000000000001F6"
-"0000000000000000000001F7"
-"0000000000000000000001F8"
-"0000000000000000000001F9"
-"0000000000000000000001FA"
-"0000000000000000000001FB"
-"0000000000000000000001FC"
-"0000000000000000000001FD"
-"0000000000000000000001FE"
-"0000000000000000000001FF"
-"000000000000000000000200"
-"000000000000000000000201"
-"000000000000000000000202"
-"000000000000000000000203"
-"000000000000000000000204"
-"000000000000000000000205"
-"000000000000000000000206"
-"000000000000000000000207"
-"000000000000000000000208"
-"000000000000000000000209"
-"00000000000000000000020A"
-"00000000000000000000020B"
-"00000000000000000000020C"
-"00000000000000000000020D"
-"00000000000000000000020E"
-"00000000000000000000020F"
-"000000000000000000000210"
-"000000000000000000000211"
-"000000000000000000000212"
-"000000000000000000000213"
-"000000000000000000000214"
-"000000000000000000000215"
-"000000000000000000000216"
-"000000000000000000000217"
-"000000000000000000000218"
-"000000000000000000000219"
-"00000000000000000000021A"
-"00000000000000000000021B"
-"00000000000000000000021C"
-"00000000000000000000021D"
-"00000000000000000000021E"
-"00000000000000000000021F"
-"000000000000000000000220"
-"000000000000000000000221"
-"000000000000000000000222"
-"000000000000000000000223"
-"000000000000000000000224"
-"000000000000000000000225"
-"000000000000000000000226"
-"000000000000000000000227"
-"000000000000000000000228"
-"000000000000000000000229"
-"00000000000000000000022A"
-"00000000000000000000022B"
-"00000000000000000000022C"
-"00000000000000000000022D"
-"00000000000000000000022E"
-"00000000000000000000022F"
-"000000000000000000000230"
-"000000000000000000000231"
-"000000000000000000000232"
-"000000000000000000000233"
-"000000000000000000000234"
-"000000000000000000000235"
-"000000000000000000000236"
-"000000000000000000000237"
-"000000000000000000000238"
-"000000000000000000000239"
-"00000000000000000000023A"
-"00000000000000000000023B"
-"00000000000000000000023C"
-"00000000000000000000023D"
-"00000000000000000000023E"
-"00000000000000000000023F"
-"000000000000000000000240"
-"000000000000000000000241"
-"000000000000000000000242"
-"000000000000000000000243"
-"000000000000000000000244"
-"000000000000000000000245"
-"000000000000000000000246"
-"000000000000000000000247"
-"000000000000000000000248"
-"000000000000000000000249"
-"00000000000000000000024A"
-"00000000000000000000024B"
-"00000000000000000000024C"
-"00000000000000000000024D"
-"00000000000000000000024E"
-"00000000000000000000024F"
-"000000000000000000000250"
-"000000000000000000000251"
-"000000000000000000000252"
-"000000000000000000000253"
-"000000000000000000000254"
-"000000000000000000000255"
-"000000000000000000000256"
-"000000000000000000000257"
-"000000000000000000000258"
-"000000000000000000000259"
-"00000000000000000000025A"
-"00000000000000000000025B"
-"00000000000000000000025C"
-"00000000000000000000025D"
-"00000000000000000000025E"
-"00000000000000000000025F"
-"000000000000000000000260"
-"000000000000000000000261"
-"000000000000000000000262"
-"000000000000000000000263"
-"000000000000000000000264"
-"000000000000000000000265"
-"000000000000000000000266"
-"000000000000000000000267"
-"000000000000000000000268"
-"000000000000000000000269"
-"00000000000000000000026A"
-"00000000000000000000026B"
-"00000000000000000000026C"
-"00000000000000000000026D"
-"00000000000000000000026E"
-"00000000000000000000026F"
-"000000000000000000000270"
-"000000000000000000000271"
-"000000000000000000000272"
-"000000000000000000000273"
-"000000000000000000000274"
-"000000000000000000000275"
-"000000000000000000000276"
-"000000000000000000000277"
-"000000000000000000000278"
-"000000000000000000000279"
-"00000000000000000000027A"
-"00000000000000000000027B"
-"00000000000000000000027C"
-"00000000000000000000027D"
-"00000000000000000000027E"
-"00000000000000000000027F"
-"000000000000000000000280"
-"000000000000000000000281"
-"000000000000000000000282"
-"000000000000000000000283"
-"000000000000000000000284"
-"000000000000000000000285"
-"000000000000000000000286"
-"000000000000000000000287"
-"000000000000000000000288"
-"000000000000000000000289"
-"00000000000000000000028A"
-"00000000000000000000028B"
-"00000000000000000000028C"
-"00000000000000000000028D"
-"00000000000000000000028E"
-"00000000000000000000028F"
-"000000000000000000000290"
-"000000000000000000000291"
-"000000000000000000000292"
-"000000000000000000000293"
-"000000000000000000000294"
-"000000000000000000000295"
-"000000000000000000000296"
-"000000000000000000000297"
-"000000000000000000000298"
-"000000000000000000000299"
-"00000000000000000000029A"
-"00000000000000000000029B"
-"00000000000000000000029C"
-"00000000000000000000029D"
-"00000000000000000000029E"
-"00000000000000000000029F"
-"0000000000000000000002A0"
-"0000000000000000000002A1"
-"0000000000000000000002A2"
-"0000000000000000000002A3"
-"0000000000000000000002A4"
-"0000000000000000000002A5"
-"0000000000000000000002A6"
-"0000000000000000000002A7"
-"0000000000000000000002A8"
-"0000000000000000000002A9"
-"0000000000000000000002AA"
-"0000000000000000000002AB"
-"0000000000000000000002AC"
-"0000000000000000000002AD"
-"0000000000000000000002AE"
-"0000000000000000000002AF"
-"0000000000000000000002B0"
-"0000000000000000000002B1"
-"0000000000000000000002B2"
-"0000000000000000000002B3"
-"0000000000000000000002B4"
-"0000000000000000000002B5"
-"0000000000000000000002B6"
-"0000000000000000000002B7"
-"0000000000000000000002B8"
-"0000000000000000000002B9"
-"0000000000000000000002BA"
-"0000000000000000000002BB"
-"0000000000000000000002BC"
-"0000000000000000000002BD"
-"0000000000000000000002BE"
-"0000000000000000000002BF"
-"0000000000000000000002C0"
-"0000000000000000000002C1"
-"0000000000000000000002C2"
-"0000000000000000000002C3"
-"0000000000000000000002C4"
-"0000000000000000000002C5"
-"0000000000000000000002C6"
-"0000000000000000000002C7"
-"0000000000000000000002C8"
-"0000000000000000000002C9"
-"0000000000000000000002CA"
-"0000000000000000000002CB"
-"0000000000000000000002CC"
-"0000000000000000000002CD"
-"0000000000000000000002CE"
-"0000000000000000000002CF"
-"0000000000000000000002D0"
-"0000000000000000000002D1"
-"0000000000000000000002D2"
-"0000000000000000000002D3"
-"0000000000000000000002D4"
-"0000000000000000000002D5"
-"0000000000000000000002D6"
-"0000000000000000000002D7"
-"0000000000000000000002D8"
-"0000000000000000000002D9"
-"0000000000000000000002DA"
-"0000000000000000000002DB"
-"0000000000000000000002DC"
-"0000000000000000000002DD"
-"0000000000000000000002DE"
-"0000000000000000000002DF"
-"0000000000000000000002E0"
-"0000000000000000000002E1"
-"0000000000000000000002E2"
-"0000000000000000000002E3"
-"0000000000000000000002E4"
-"0000000000000000000002E5"
-"0000000000000000000002E6"
-"0000000000000000000002E7"
-"0000000000000000000002E8"
-"0000000000000000000002E9"
-"0000000000000000000002EA"
-"0000000000000000000002EB"
-"0000000000000000000002EC"
-"0000000000000000000002ED"
-"0000000000000000000002EE"
-"0000000000000000000002EF"
-"0000000000000000000002F0"
-"0000000000000000000002F1"
-"0000000000000000000002F2"
-"0000000000000000000002F3"
-"0000000000000000000002F4"
-"0000000000000000000002F5"
-"0000000000000000000002F6"
-"0000000000000000000002F7"
-"0000000000000000000002F8"
-"0000000000000000000002F9"
-"0000000000000000000002FA"
-"0000000000000000000002FB"
-"0000000000000000000002FC"
-"0000000000000000000002FD"
-"0000000000000000000002FE"
-"0000000000000000000002FF"
-"000000000000000000000300"
-"000000000000000000000301"
-"000000000000000000000302"
-"000000000000000000000303"
-"000000000000000000000304"
-"000000000000000000000305"
-"000000000000000000000306"
-"000000000000000000000307"
-"000000000000000000000308"
-"000000000000000000000309"
-"00000000000000000000030A"
-"00000000000000000000030B"
-"00000000000000000000030C"
-"00000000000000000000030D"
-"00000000000000000000030E"
-"00000000000000000000030F"
-"000000000000000000000310"
-"000000000000000000000311"
-"000000000000000000000312"
-"000000000000000000000313"
-"000000000000000000000314"
-"000000000000000000000315"
-"000000000000000000000316"
-"000000000000000000000317"
-"000000000000000000000318"
-"000000000000000000000319"
-"00000000000000000000031A"
-"00000000000000000000031B"
-"00000000000000000000031C"
-"00000000000000000000031D"
-"00000000000000000000031E"
-"00000000000000000000031F"
-"000000000000000000000320"
-"000000000000000000000321"
-"000000000000000000000322"
-"000000000000000000000323"
-"000000000000000000000324"
-"000000000000000000000325"
-"000000000000000000000326"
-"000000000000000000000327"
-"000000000000000000000328"
-"000000000000000000000329"
-"00000000000000000000032A"
-"00000000000000000000032B"
-"00000000000000000000032C"
-"00000000000000000000032D"
-"00000000000000000000032E"
-"00000000000000000000032F"
-"000000000000000000000330"
-"000000000000000000000331"
-"000000000000000000000332"
-"000000000000000000000333"
-"000000000000000000000334"
-"000000000000000000000335"
-"000000000000000000000336"
-"000000000000000000000337"
-"000000000000000000000338"
-"000000000000000000000339"
-"00000000000000000000033A"
-"00000000000000000000033B"
-"00000000000000000000033C"
-"00000000000000000000033D"
-"00000000000000000000033E"
-"00000000000000000000033F"
-"000000000000000000000340"
-"000000000000000000000341"
-"000000000000000000000342"
-"000000000000000000000343"
-"000000000000000000000344"
-"000000000000000000000345"
-"000000000000000000000346"
-"000000000000000000000347"
-"000000000000000000000348"
-"000000000000000000000349"
-"00000000000000000000034A"
-"00000000000000000000034B"
-"00000000000000000000034C"
-"00000000000000000000034D"
-"00000000000000000000034E"
-"00000000000000000000034F"
-"000000000000000000000350"
-"000000000000000000000351"
-"000000000000000000000352"
-"000000000000000000000353"
-"000000000000000000000354"
-"000000000000000000000355"
-"000000000000000000000356"
-"000000000000000000000357"
-"000000000000000000000358"
-"000000000000000000000359"
-"00000000000000000000035A"
-"00000000000000000000035B"
-"00000000000000000000035C"
-"00000000000000000000035D"
-"00000000000000000000035E"
-"00000000000000000000035F"
-"000000000000000000000360"
-"000000000000000000000361"
-"000000000000000000000362"
-"000000000000000000000363"
-"000000000000000000000364"
-"000000000000000000000365"
-"000000000000000000000366"
-"000000000000000000000367"
-"000000000000000000000368"
-"000000000000000000000369"
-"00000000000000000000036A"
-"00000000000000000000036B"
-"00000000000000000000036C"
-"00000000000000000000036D"
-"00000000000000000000036E"
-"00000000000000000000036F"
-"000000000000000000000370"
-"000000000000000000000371"
-"000000000000000000000372"
-"000000000000000000000373"
-"000000000000000000000374"
-"000000000000000000000375"
-"000000000000000000000376"
-"000000000000000000000377"
-"000000000000000000000378"
-"000000000000000000000379"
-"00000000000000000000037A"
-"00000000000000000000037B"
-"00000000000000000000037C"
-"00000000000000000000037D"
-"00000000000000000000037E"
-"00000000000000000000037F"
-"000000000000000000000380"
-"000000000000000000000381"
-"000000000000000000000382"
-"000000000000000000000383"
-"000000000000000000000384"
-"000000000000000000000385"
-"000000000000000000000386"
-"000000000000000000000387"
-"000000000000000000000388"
-"000000000000000000000389"
-"00000000000000000000038A"
-"00000000000000000000038B"
-"00000000000000000000038C"
-"00000000000000000000038D"
-"00000000000000000000038E"
-"00000000000000000000038F"
-"000000000000000000000390"
-"000000000000000000000391"
-"000000000000000000000392"
-"000000000000000000000393"
-"000000000000000000000394"
-"000000000000000000000395"
-"000000000000000000000396"
-"000000000000000000000397"
-"000000000000000000000398"
-"000000000000000000000399"
-"00000000000000000000039A"
-"00000000000000000000039B"
-"00000000000000000000039C"
-"00000000000000000000039D"
-"00000000000000000000039E"
-"00000000000000000000039F"
-"0000000000000000000003A0"
-"0000000000000000000003A1"
-"0000000000000000000003A2"
-"0000000000000000000003A3"
-"0000000000000000000003A4"
-"0000000000000000000003A5"
-"0000000000000000000003A6"
-"0000000000000000000003A7"
-"0000000000000000000003A8"
-"0000000000000000000003A9"
-"0000000000000000000003AA"
-"0000000000000000000003AB"
-"0000000000000000000003AC"
-"0000000000000000000003AD"
-"0000000000000000000003AE"
-"0000000000000000000003AF"
-"0000000000000000000003B0"
-"0000000000000000000003B1"
-"0000000000000000000003B2"
-"0000000000000000000003B3"
-"0000000000000000000003B4"
-"0000000000000000000003B5"
-"0000000000000000000003B6"
-"0000000000000000000003B7"
-"0000000000000000000003B8"
-"0000000000000000000003B9"
-"0000000000000000000003BA"
-"0000000000000000000003BB"
-"0000000000000000000003BC"
-"0000000000000000000003BD"
-"0000000000000000000003BE"
-"0000000000000000000003BF"
-"0000000000000000000003C0"
-"0000000000000000000003C1"
-"0000000000000000000003C2"
-"0000000000000000000003C3"
-"0000000000000000000003C4"
-"0000000000000000000003C5"
-"0000000000000000000003C6"
-"0000000000000000000003C7"
-"0000000000000000000003C8"
-"0000000000000000000003C9"
-"0000000000000000000003CA"
-"0000000000000000000003CB"
-"0000000000000000000003CC"
-"0000000000000000000003CD"
-"0000000000000000000003CE"
-"0000000000000000000003CF"
-"0000000000000000000003D0"
-"0000000000000000000003D1"
-"0000000000000000000003D2"
-"0000000000000000000003D3"
-"0000000000000000000003D4"
-"0000000000000000000003D5"
-"0000000000000000000003D6"
-"0000000000000000000003D7"
-"0000000000000000000003D8"
-"0000000000000000000003D9"
-"0000000000000000000003DA"
-"0000000000000000000003DB"
-"0000000000000000000003DC"
-"0000000000000000000003DD"
-"0000000000000000000003DE"
-"0000000000000000000003DF"
-"0000000000000000000003E0"
-"0000000000000000000003E1"
-"0000000000000000000003E2"
-"0000000000000000000003E3"
-"0000000000000000000003E4"
-"0000000000000000000003E5"
-"0000000000000000000003E6"
-"0000000000000000000003E7"
-"0000000000000000000003E8"
-"0000000000000000000003E9"
-"0000000000000000000003EA"
-"0000000000000000000003EB"
-"0000000000000000000003EC"
-"0000000000000000000003ED"
-"0000000000000000000003EE"
-"0000000000000000000003EF"
-"0000000000000000000003F0"
-"0000000000000000000003F1"
-"0000000000000000000003F2"
-"0000000000000000000003F3"
-"0000000000000000000003F4"
-"0000000000000000000003F5"
-"0000000000000000000003F6"
-"0000000000000000000003F7"
-"0000000000000000000003F8"
-"0000000000000000000003F9"
-"0000000000000000000003FA"
-"0000000000000000000003FB"
-"0000000000000000000003FC"
-"0000000000000000000003FD"
-"0000000000000000000003FE"
-"0000000000000000000003FF"
-"000000000000000000000400"
-"000000000000000000000401"
-"000000000000000000000402"
-"000000000000000000000403"
-"000000000000000000000404"
-"000000000000000000000405"
-"000000000000000000000406"
-"000000000000000000000407"
-"000000000000000000000408"
-"000000000000000000000409"
-"00000000000000000000040A"
-"00000000000000000000040B"
-"00000000000000000000040C"
-"00000000000000000000040D"
-"00000000000000000000040E"
-"00000000000000000000040F"
-"000000000000000000000410"
-"000000000000000000000411"
-"000000000000000000000412"
-"000000000000000000000413"
-"000000000000000000000414"
-"000000000000000000000415"
-"000000000000000000000416"
-"000000000000000000000417"
-"000000000000000000000418"
-"000000000000000000000419"
-"00000000000000000000041A"
-"00000000000000000000041B"
-"00000000000000000000041C"
-"00000000000000000000041D"
-"00000000000000000000041E"
-"00000000000000000000041F"
-"000000000000000000000420"
-"000000000000000000000421"
-"000000000000000000000422"
-"000000000000000000000423"
-"000000000000000000000424"
-"000000000000000000000425"
-"000000000000000000000426"
-"000000000000000000000427"
-"000000000000000000000428"
-"000000000000000000000429"
-"00000000000000000000042A"
-"00000000000000000000042B"
-"00000000000000000000042C"
-"00000000000000000000042D"
-"00000000000000000000042E"
-"00000000000000000000042F"
-"000000000000000000000430"
-"000000000000000000000431"
-"000000000000000000000432"
-"000000000000000000000433"
-"000000000000000000000434"
-"000000000000000000000435"
-"000000000000000000000436"
-"000000000000000000000437"
-"000000000000000000000438"
-"000000000000000000000439"
-"00000000000000000000043A"
-"00000000000000000000043B"
-"00000000000000000000043C"
-"00000000000000000000043D"
-"00000000000000000000043E"
-"00000000000000000000043F"
-"000000000000000000000440"
-"000000000000000000000441"
-"000000000000000000000442"
-"000000000000000000000443"
-"000000000000000000000444"
-"000000000000000000000445"
-"000000000000000000000446"
-"000000000000000000000447"
-"000000000000000000000448"
-"000000000000000000000449"
-"00000000000000000000044A"
-"00000000000000000000044B"
-"00000000000000000000044C"
-"00000000000000000000044D"
-"00000000000000000000044E"
-"00000000000000000000044F"
-"000000000000000000000450"
-"000000000000000000000451"
-"000000000000000000000452"
-"000000000000000000000453"
-"000000000000000000000454"
-"000000000000000000000455"
-"000000000000000000000456"
-"000000000000000000000457"
-"000000000000000000000458"
-"000000000000000000000459"
-"00000000000000000000045A"
-"00000000000000000000045B"
-"00000000000000000000045C"
-"00000000000000000000045D"
-"00000000000000000000045E"
-"00000000000000000000045F"
-"000000000000000000000460"
-"000000000000000000000461"
-"000000000000000000000462"
-"000000000000000000000463"
-"000000000000000000000464"
-"000000000000000000000465"
-"000000000000000000000466"
-"000000000000000000000467"
-"000000000000000000000468"
-"000000000000000000000469"
-"00000000000000000000046A"
-"00000000000000000000046B"
-"00000000000000000000046C"
-"00000000000000000000046D"
-"00000000000000000000046E"
-"00000000000000000000046F"
-"000000000000000000000470"
-"000000000000000000000471"
-"000000000000000000000472"
-"000000000000000000000473"
-"000000000000000000000474"
-"000000000000000000000475"
-"000000000000000000000476"
-"000000000000000000000477"
-"000000000000000000000478"
-"000000000000000000000479"
-"00000000000000000000047A"
-"00000000000000000000047B"
-"00000000000000000000047C"
-"00000000000000000000047D"
-"00000000000000000000047E"
-"00000000000000000000047F"
-"000000000000000000000480"
-"000000000000000000000481"
-"000000000000000000000482"
-"000000000000000000000483"
-"000000000000000000000484"
-"000000000000000000000485"
-"000000000000000000000486"
-"000000000000000000000487"
-"000000000000000000000488"
-"000000000000000000000489"
-"00000000000000000000048A"
-"00000000000000000000048B"
-"00000000000000000000048C"
-"00000000000000000000048D"
-"00000000000000000000048E"
-"00000000000000000000048F"
-"000000000000000000000490"
-"000000000000000000000491"
-"000000000000000000000492"
-"000000000000000000000493"
-"000000000000000000000494"
-"000000000000000000000495"
-"000000000000000000000496"
-"000000000000000000000497"
-"000000000000000000000498"
-"000000000000000000000499"
-"00000000000000000000049A"
-"00000000000000000000049B"
-"00000000000000000000049C"
-"00000000000000000000049D"
-"00000000000000000000049E"
-"00000000000000000000049F"
-"0000000000000000000004A0"
-"0000000000000000000004A1"
-"0000000000000000000004A2"
-"0000000000000000000004A3"
-"0000000000000000000004A4"
-"0000000000000000000004A5"
-"0000000000000000000004A6"
-"0000000000000000000004A7"
-"0000000000000000000004A8"
-"0000000000000000000004A9"
-"0000000000000000000004AA"
-"0000000000000000000004AB"
-"0000000000000000000004AC"
-"0000000000000000000004AD"
-"0000000000000000000004AE"
-"0000000000000000000004AF"
-"0000000000000000000004B0"
-"0000000000000000000004B1"
-"0000000000000000000004B2"
-"0000000000000000000004B3"
-"0000000000000000000004B4"
-"0000000000000000000004B5"
-"0000000000000000000004B6"
-"0000000000000000000004B7"
-"0000000000000000000004B8"
-"0000000000000000000004B9"
-"0000000000000000000004BA"
-"0000000000000000000004BB"
-"0000000000000000000004BC"
-"0000000000000000000004BD"
-"0000000000000000000004BE"
-"0000000000000000000004BF"
-"0000000000000000000004C0"
-"0000000000000000000004C1"
-"0000000000000000000004C2"
-"0000000000000000000004C3"
-"0000000000000000000004C4"
-"0000000000000000000004C5"
-"0000000000000000000004C6"
-"0000000000000000000004C7"
-"0000000000000000000004C8"
-"0000000000000000000004C9"
-"0000000000000000000004CA"
-"0000000000000000000004CB"
-"0000000000000000000004CC"
-"0000000000000000000004CD"
-"0000000000000000000004CE"
-"0000000000000000000004CF"
-"0000000000000000000004D0"
-"0000000000000000000004D1"
-"0000000000000000000004D2"
-"0000000000000000000004D3"
-"0000000000000000000004D4"
-"0000000000000000000004D5"
-"0000000000000000000004D6"
-"0000000000000000000004D7"
-"0000000000000000000004D8"
-"0000000000000000000004D9"
-"0000000000000000000004DA"
-"0000000000000000000004DB"
-"0000000000000000000004DC"
-"0000000000000000000004DD"
-"0000000000000000000004DE"
-"0000000000000000000004DF"
-"0000000000000000000004E0"
-"0000000000000000000004E1"
-"0000000000000000000004E2"
-"0000000000000000000004E3"
-"0000000000000000000004E4"
-"0000000000000000000004E5"
-"0000000000000000000004E6"
-"0000000000000000000004E7"
-"0000000000000000000004E8"
-"0000000000000000000004E9"
-"0000000000000000000004EA"
-"0000000000000000000004EB"
-"0000000000000000000004EC"
-"0000000000000000000004ED"
-"0000000000000000000004EE"
-"0000000000000000000004EF"
-"0000000000000000000004F0"
-"0000000000000000000004F1"
-"0000000000000000000004F2"
-"0000000000000000000004F3"
-"0000000000000000000004F4"
-"0000000000000000000004F5"
-"0000000000000000000004F6"
-"0000000000000000000004F7"
-"0000000000000000000004F8"
-"0000000000000000000004F9"
-"0000000000000000000004FA"
-"0000000000000000000004FB"
-"0000000000000000000004FC"
-"0000000000000000000004FD"
-"0000000000000000000004FE"
-"0000000000000000000004FF"
-"000000000000000000000500"
-"000000000000000000000501"
-"000000000000000000000502"
-"000000000000000000000503"
-"000000000000000000000504"
-"000000000000000000000505"
-"000000000000000000000506"
-"000000000000000000000507"
-"000000000000000000000508"
-"000000000000000000000509"
-"00000000000000000000050A"
-"00000000000000000000050B"
-"00000000000000000000050C"
-"00000000000000000000050D"
-"00000000000000000000050E"
-"00000000000000000000050F"
-"000000000000000000000510"
-"000000000000000000000511"
-"000000000000000000000512"
-"000000000000000000000513"
-"000000000000000000000514"
-"000000000000000000000515"
-"000000000000000000000516"
-"000000000000000000000517"
-"000000000000000000000518"
-"000000000000000000000519"
-"00000000000000000000051A"
-"00000000000000000000051B"
-"00000000000000000000051C"
-"00000000000000000000051D"
-"00000000000000000000051E"
-"00000000000000000000051F"
-"000000000000000000000520"
-"000000000000000000000521"
-"000000000000000000000522"
-"000000000000000000000523"
-"000000000000000000000524"
-"000000000000000000000525"
-"000000000000000000000526"
-"000000000000000000000527"
-"000000000000000000000528"
-"000000000000000000000529"
-"00000000000000000000052A"
-"00000000000000000000052B"
-"00000000000000000000052C"
-"00000000000000000000052D"
-"00000000000000000000052E"
-"00000000000000000000052F"
-"000000000000000000000530"
-"000000000000000000000531"
-"000000000000000000000532"
-"000000000000000000000533"
-"000000000000000000000534"
-"000000000000000000000535"
-"000000000000000000000536"
-"000000000000000000000537"
-"000000000000000000000538"
-"000000000000000000000539"
-"00000000000000000000053A"
-"00000000000000000000053B"
-"00000000000000000000053C"
-"00000000000000000000053D"
-"00000000000000000000053E"
-"00000000000000000000053F"
-"000000000000000000000540"
-"000000000000000000000541"
-"000000000000000000000542"
-"000000000000000000000543"
-"000000000000000000000544"
-"000000000000000000000545"
-"000000000000000000000546"
-"000000000000000000000547"
-"000000000000000000000548"
-"000000000000000000000549"
-"00000000000000000000054A"
-"00000000000000000000054B"
-"00000000000000000000054C"
-"00000000000000000000054D"
-"00000000000000000000054E"
-"00000000000000000000054F"
-"000000000000000000000550"
-"000000000000000000000551"
-"000000000000000000000552"
-"000000000000000000000553"
-"000000000000000000000554"
-"000000000000000000000555"
-"000000000000000000000556"
-"000000000000000000000557"
-"000000000000000000000558"
-"000000000000000000000559"
-"00000000000000000000055A"
-"00000000000000000000055B"
-"00000000000000000000055C"
-"00000000000000000000055D"
-"00000000000000000000055E"
-"00000000000000000000055F"
-"000000000000000000000560"
-"000000000000000000000561"
-"000000000000000000000562"
-"000000000000000000000563"
-"000000000000000000000564"
-"000000000000000000000565"
-"000000000000000000000566"
-"000000000000000000000567"
-"000000000000000000000568"
-"000000000000000000000569"
-"00000000000000000000056A"
-"00000000000000000000056B"
-"00000000000000000000056C"
-"00000000000000000000056D"
-"00000000000000000000056E"
-"00000000000000000000056F"
-"000000000000000000000570"
-"000000000000000000000571"
-"000000000000000000000572"
-"000000000000000000000573"
-"000000000000000000000574"
-"000000000000000000000575"
-"000000000000000000000576"
-"000000000000000000000577"
-"000000000000000000000578"
-"000000000000000000000579"
-"00000000000000000000057A"
-"00000000000000000000057B"
-"00000000000000000000057C"
-"00000000000000000000057D"
-"00000000000000000000057E"
-"00000000000000000000057F"
-"000000000000000000000580"
-"000000000000000000000581"
-"000000000000000000000582"
-"000000000000000000000583"
-"000000000000000000000584"
-"000000000000000000000585"
-"000000000000000000000586"
-"000000000000000000000587"
-"000000000000000000000588"
-"000000000000000000000589"
-"00000000000000000000058A"
-"00000000000000000000058B"
-"00000000000000000000058C"
-"00000000000000000000058D"
-"00000000000000000000058E"
-"00000000000000000000058F"
-"000000000000000000000590"
-"000000000000000000000591"
-"000000000000000000000592"
-"000000000000000000000593"
-"000000000000000000000594"
-"000000000000000000000595"
-"000000000000000000000596"
-"000000000000000000000597"
-"000000000000000000000598"
-"000000000000000000000599"
-"00000000000000000000059A"
-"00000000000000000000059B"
-"00000000000000000000059C"
-"00000000000000000000059D"
-"00000000000000000000059E"
-"00000000000000000000059F"
-"0000000000000000000005A0"
-"0000000000000000000005A1"
-"0000000000000000000005A2"
-"0000000000000000000005A3"
-"0000000000000000000005A4"
-"0000000000000000000005A5"
-"0000000000000000000005A6"
-"0000000000000000000005A7"
-"0000000000000000000005A8"
-"0000000000000000000005A9"
-"0000000000000000000005AA"
-"0000000000000000000005AB"
-"0000000000000000000005AC"
-"0000000000000000000005AD"
-"0000000000000000000005AE"
-"0000000000000000000005AF"
-"0000000000000000000005B0"
-"0000000000000000000005B1"
-"0000000000000000000005B2"
-"0000000000000000000005B3"
-"0000000000000000000005B4"
-"0000000000000000000005B5"
-"0000000000000000000005B6"
-"0000000000000000000005B7"
-"0000000000000000000005B8"
-"0000000000000000000005B9"
-"0000000000000000000005BA"
-"0000000000000000000005BB"
-"0000000000000000000005BC"
-"0000000000000000000005BD"
-"0000000000000000000005BE"
-"0000000000000000000005BF"
-"0000000000000000000005C0"
-"0000000000000000000005C1"
-"0000000000000000000005C2"
-"0000000000000000000005C3"
-"0000000000000000000005C4"
-"0000000000000000000005C5"
-"0000000000000000000005C6"
-"0000000000000000000005C7"
-"0000000000000000000005C8"
-"0000000000000000000005C9"
-"0000000000000000000005CA"
-"0000000000000000000005CB"
-"0000000000000000000005CC"
-"0000000000000000000005CD"
-"0000000000000000000005CE"
-"0000000000000000000005CF"
-"0000000000000000000005D0"
-"0000000000000000000005D1"
-"0000000000000000000005D2"
-"0000000000000000000005D3"
-"0000000000000000000005D4"
-"0000000000000000000005D5"
-"0000000000000000000005D6"
-"0000000000000000000005D7"
-"0000000000000000000005D8"
-"0000000000000000000005D9"
-"0000000000000000000005DA"
-"0000000000000000000005DB"
-"0000000000000000000005DC"
-"0000000000000000000005DD"
-"0000000000000000000005DE"
-"0000000000000000000005DF"
-"0000000000000000000005E0"
-"0000000000000000000005E1"
-"0000000000000000000005E2"
-"0000000000000000000005E3"
-"0000000000000000000005E4"
-"0000000000000000000005E5"
-"0000000000000000000005E6"
-"0000000000000000000005E7"
-"000000010000000000000001"
-"000000010000000000000002"
-"000000010000000000000003"
-"000000010000000000000004"
-"000000010000000000000005"
-"000000010000000000000006"
-"000000010000000000000007"
-"000000010000000000000008"
-"000000010000000000000009"
-"00000001000000000000000A"
-"00000001000000000000000B"
-"00000001000000000000000C"
-"00000001000000000000000D"
-"00000001000000000000000E"
-"00000001000000000000000F"
-"000000010000000000000010"
-"000000010000000000000011"
-"000000010000000000000012"
-"000000010000000000000013"
-"000000010000000000000014"
-"000000010000000000000015"
-"000000010000000000000016"
-"000000010000000000000017"
-"000000010000000000000018"
-"000000010000000000000019"
-"00000001000000000000001A"
-"00000001000000000000001B"
-"00000001000000000000001C"
-"00000001000000000000001D"
-"00000001000000000000001E"
-"00000001000000000000001F"
-"000000010000000000000020"
-"000000010000000000000021"
-"000000010000000000000022"
-"000000010000000000000023"
-"000000010000000000000024"
-"000000010000000000000025"
-"000000010000000000000026"
-"000000010000000000000027"
-"000000010000000000000028"
-"000000010000000000000029"
-"00000001000000000000002A"
-"00000001000000000000002B"
-"00000001000000000000002C"
-"00000001000000000000002D"
-"00000001000000000000002E"
-"00000001000000000000002F"
-"000000010000000000000030"
-"000000010000000000000031"
-"000000010000000000000032"
-"000000010000000000000033"
-"000000010000000000000034"
-"000000010000000000000035"
-"000000010000000000000036"
-"000000010000000000000037"
-"000000010000000000000038"
-"000000010000000000000039"
-"00000001000000000000003A"
-"00000001000000000000003B"
-"00000001000000000000003C"
-"00000001000000000000003D"
-"00000001000000000000003E"
-"00000001000000000000003F"
-"000000010000000000000040"
-"000000010000000000000041"
-"000000010000000000000042"
-"000000010000000000000043"
-"000000010000000000000044"
-"000000010000000000000045"
-"000000010000000000000046"
-"000000010000000000000047"
-"000000010000000000000048"
-"000000010000000000000049"
-"00000001000000000000004A"
-"00000001000000000000004B"
-"00000001000000000000004C"
-"00000001000000000000004D"
-"00000001000000000000004E"
-"00000001000000000000004F"
-"000000010000000000000050"
-"000000010000000000000051"
-"000000010000000000000052"
-"000000010000000000000053"
-"000000010000000000000054"
-"000000010000000000000055"
-"000000010000000000000056"
-"000000010000000000000057"
-"000000010000000000000058"
-"000000010000000000000059"
-"00000001000000000000005A"
-"00000001000000000000005B"
-"00000001000000000000005C"
-"00000001000000000000005D"
-"00000001000000000000005E"
-"00000001000000000000005F"
-"000000010000000000000060"
-"000000010000000000000061"
-"000000010000000000000062"
-"000000010000000000000063"
-"000000010000000000000064"
-"000000010000000000000065"
-"000000010000000000000066"
-"000000010000000000000067"
-"000000010000000000000068"
-"000000010000000000000069"
-"00000001000000000000006A"
-"00000001000000000000006B"
-"00000001000000000000006C"
-"00000001000000000000006D"
-"00000001000000000000006E"
-"00000001000000000000006F"
-"000000010000000000000070"
-"000000010000000000000071"
-"000000010000000000000072"
-"000000010000000000000073"
-"000000010000000000000074"
-"000000010000000000000075"
-"000000010000000000000076"
-"000000010000000000000077"
-"000000010000000000000078"
-"000000010000000000000079"
-"00000001000000000000007A"
-"00000001000000000000007B"
-"00000001000000000000007C"
-"00000001000000000000007D"
-"00000001000000000000007E"
-"00000001000000000000007F"
-"000000010000000000000080"
-"000000010000000000000081"
-"000000010000000000000082"
-"000000010000000000000083"
-"000000010000000000000084"
-"000000010000000000000085"
-"000000010000000000000086"
-"000000010000000000000087"
-"000000010000000000000088"
-"000000010000000000000089"
-"00000001000000000000008A"
-"00000001000000000000008B"
-"00000001000000000000008C"
-"00000001000000000000008D"
-"00000001000000000000008E"
-"00000001000000000000008F"
-"000000010000000000000090"
-"000000010000000000000091"
-"000000010000000000000092"
-"000000010000000000000093"
-"000000010000000000000094"
-"000000010000000000000095"
-"000000010000000000000096"
-"000000010000000000000097"
-"000000010000000000000098"
-"000000010000000000000099"
-"00000001000000000000009A"
-"00000001000000000000009B"
-"00000001000000000000009C"
-"00000001000000000000009D"
-"00000001000000000000009E"
-"00000001000000000000009F"
-"0000000100000000000000A0"
-"0000000100000000000000A1"
-"0000000100000000000000A2"
-"0000000100000000000000A3"
-"0000000100000000000000A4"
-"0000000100000000000000A5"
-"0000000100000000000000A6"
-"0000000100000000000000A7"
-"0000000100000000000000A8"
-"0000000100000000000000A9"
-"0000000100000000000000AA"
-"0000000100000000000000AB"
-"0000000100000000000000AC"
-"0000000100000000000000AD"
-"0000000100000000000000AE"
-"0000000100000000000000AF"
-"0000000100000000000000B0"
-"0000000100000000000000B1"
-"0000000100000000000000B2"
-"0000000100000000000000B3"
-"0000000100000000000000B4"
-"0000000100000000000000B5"
-"0000000100000000000000B6"
-"0000000100000000000000B7"
-"0000000100000000000000B8"
-"0000000100000000000000B9"
-"0000000100000000000000BA"
-"0000000100000000000000BB"
-"0000000100000000000000BC"
-"0000000100000000000000BD"
-"0000000100000000000000BE"
-"0000000100000000000000BF"
-"0000000100000000000000C0"
-"0000000100000000000000C1"
-"0000000100000000000000C2"
-"0000000100000000000000C3"
-"0000000100000000000000C4"
-"0000000100000000000000C5"
-"0000000100000000000000C6"
-"0000000100000000000000C7"
-"0000000100000000000000C8"
-"0000000100000000000000C9"
-"0000000100000000000000CA"
-"0000000100000000000000CB"
-"0000000100000000000000CC"
-"0000000100000000000000CD"
-"0000000100000000000000CE"
-"0000000100000000000000CF"
-"0000000100000000000000D0"
-"0000000100000000000000D1"
-"0000000100000000000000D2"
-"0000000100000000000000D3"
-"0000000100000000000000D4"
-"0000000100000000000000D5"
-"0000000100000000000000D6"
-"0000000100000000000000D7"
-"0000000100000000000000D8"
-"0000000100000000000000D9"
-"0000000100000000000000DA"
-"0000000100000000000000DB"
-"0000000100000000000000DC"
-"0000000100000000000000DD"
-"0000000100000000000000DE"
-"0000000100000000000000DF"
-"0000000100000000000000E0"
-"0000000100000000000000E1"
-"0000000100000000000000E2"
-"0000000100000000000000E3"
-"0000000100000000000000E4"
-"0000000100000000000000E5"
-"0000000100000000000000E6"
-"0000000100000000000000E7"
-"0000000100000000000000E8"
-"0000000100000000000000E9"
-"0000000100000000000000EA"
-"0000000100000000000000EB"
-"0000000100000000000000EC"
-"0000000100000000000000ED"
-"0000000100000000000000EE"
-"0000000100000000000000EF"
-"0000000100000000000000F0"
-"0000000100000000000000F1"
-"0000000100000000000000F2"
-"0000000100000000000000F3"
-"0000000100000000000000F4"
-"0000000100000000000000F5"
-"0000000100000000000000F6"
-"0000000100000000000000F7"
-"0000000100000000000000F8"
-"0000000100000000000000F9"
-"0000000100000000000000FA"
-"0000000100000000000000FB"
-"0000000100000000000000FC"
-"0000000100000000000000FD"
-"0000000100000000000000FE"
-"0000000100000000000000FF"
-"000000010000000000000100"
-"000000010000000000000101"
-"000000010000000000000102"
-"000000010000000000000103"
-"000000010000000000000104"
-"000000010000000000000105"
-"000000010000000000000106"
-"000000010000000000000107"
-"000000010000000000000108"
-"000000010000000000000109"
-"00000001000000000000010A"
-"00000001000000000000010B"
-"00000001000000000000010C"
-"00000001000000000000010D"
-"00000001000000000000010E"
-"00000001000000000000010F"
-"000000010000000000000110"
-"000000010000000000000111"
-"000000010000000000000112"
-"000000010000000000000113"
-"000000010000000000000114"
-"000000010000000000000115"
-"000000010000000000000116"
-"000000010000000000000117"
-"000000010000000000000118"
-"000000010000000000000119"
-"00000001000000000000011A"
-"00000001000000000000011B"
-"00000001000000000000011C"
-"00000001000000000000011D"
-"00000001000000000000011E"
-"00000001000000000000011F"
-"000000010000000000000120"
-"000000010000000000000121"
-"000000010000000000000122"
-"000000010000000000000123"
-"000000010000000000000124"
-"000000010000000000000125"
-"000000010000000000000126"
-"000000010000000000000127"
-"000000010000000000000128"
-"000000010000000000000129"
-"00000001000000000000012A"
-"00000001000000000000012B"
-"00000001000000000000012C"
-"00000001000000000000012D"
-"00000001000000000000012E"
-"00000001000000000000012F"
-"000000010000000000000130"
-"000000010000000000000131"
-"000000010000000000000132"
-"000000010000000000000133"
-"000000010000000000000134"
-"000000010000000000000135"
-"000000010000000000000136"
-"000000010000000000000137"
-"000000010000000000000138"
-"000000010000000000000139"
-"00000001000000000000013A"
-"00000001000000000000013B"
-"00000001000000000000013C"
-"00000001000000000000013D"
-"00000001000000000000013E"
-"00000001000000000000013F"
-"000000010000000000000140"
-"000000010000000000000141"
-"000000010000000000000142"
-"000000010000000000000143"
-"000000010000000000000144"
-"000000010000000000000145"
-"000000010000000000000146"
-"000000010000000000000147"
-"000000010000000000000148"
-"000000010000000000000149"
-"00000001000000000000014A"
-"00000001000000000000014B"
-"00000001000000000000014C"
-"00000001000000000000014D"
-"00000001000000000000014E"
-"00000001000000000000014F"
-"000000010000000000000150"
-"000000010000000000000151"
-"000000010000000000000152"
-"000000010000000000000153"
-"000000010000000000000154"
-"000000010000000000000155"
-"000000010000000000000156"
-"000000010000000000000157"
-"000000010000000000000158"
-"000000010000000000000159"
-"00000001000000000000015A"
-"00000001000000000000015B"
-"00000001000000000000015C"
-"00000001000000000000015D"
-"00000001000000000000015E"
-"00000001000000000000015F"
-"000000010000000000000160"
-"000000010000000000000161"
-"000000010000000000000162"
-"000000010000000000000163"
-"000000010000000000000164"
-"000000010000000000000165"
-"000000010000000000000166"
-"000000010000000000000167"
-"000000010000000000000168"
-"000000010000000000000169"
-"00000001000000000000016A"
-"00000001000000000000016B"
-"00000001000000000000016C"
-"00000001000000000000016D"
-"00000001000000000000016E"
-"00000001000000000000016F"
-"000000010000000000000170"
-"000000010000000000000171"
-"000000010000000000000172"
-"000000010000000000000173"
-"000000010000000000000174"
-"000000010000000000000175"
-"000000010000000000000176"
-"000000010000000000000177"
-"000000010000000000000178"
-"000000010000000000000179"
-"00000001000000000000017A"
-"00000001000000000000017B"
-"00000001000000000000017C"
-"00000001000000000000017D"
-"00000001000000000000017E"
-"00000001000000000000017F"
-"000000010000000000000180"
-"000000010000000000000181"
-"000000010000000000000182"
-"000000010000000000000183"
-"000000010000000000000184"
-"000000010000000000000185"
-"000000010000000000000186"
-"000000010000000000000187"
-"000000010000000000000188"
-"000000010000000000000189"
-"00000001000000000000018A"
-"00000001000000000000018B"
-"00000001000000000000018C"
-"00000001000000000000018D"
-"00000001000000000000018E"
-"00000001000000000000018F"
-"000000010000000000000190"
-"000000010000000000000191"
-"000000010000000000000192"
-"000000010000000000000193"
-"000000010000000000000194"
-"000000010000000000000195"
-"000000010000000000000196"
-"000000010000000000000197"
-"000000010000000000000198"
-"000000010000000000000199"
-"00000001000000000000019A"
-"00000001000000000000019B"
-"00000001000000000000019C"
-"00000001000000000000019D"
-"00000001000000000000019E"
-"00000001000000000000019F"
-"0000000100000000000001A0"
-"0000000100000000000001A1"
-"0000000100000000000001A2"
-"0000000100000000000001A3"
-"0000000100000000000001A4"
-"0000000100000000000001A5"
-"0000000100000000000001A6"
-"0000000100000000000001A7"
-"0000000100000000000001A8"
-"0000000100000000000001A9"
-"0000000100000000000001AA"
-"0000000100000000000001AB"
-"0000000100000000000001AC"
-"0000000100000000000001AD"
-"0000000100000000000001AE"
-"0000000100000000000001AF"
-"0000000100000000000001B0"
-"0000000100000000000001B1"
-"0000000100000000000001B2"
-"0000000100000000000001B3"
-"0000000100000000000001B4"
-"0000000100000000000001B5"
-"0000000100000000000001B6"
-"0000000100000000000001B7"
-"0000000100000000000001B8"
-"0000000100000000000001B9"
-"0000000100000000000001BA"
-"0000000100000000000001BB"
-"0000000100000000000001BC"
-"0000000100000000000001BD"
-"0000000100000000000001BE"
-"0000000100000000000001BF"
-"0000000100000000000001C0"
-"0000000100000000000001C1"
-"0000000100000000000001C2"
-"0000000100000000000001C3"
-"0000000100000000000001C4"
-"0000000100000000000001C5"
-"0000000100000000000001C6"
-"0000000100000000000001C7"
-"0000000100000000000001C8"
-"0000000100000000000001C9"
-"0000000100000000000001CA"
-"0000000100000000000001CB"
-"0000000100000000000001CC"
-"0000000100000000000001CD"
-"0000000100000000000001CE"
-"0000000100000000000001CF"
-"0000000100000000000001D0"
-"0000000100000000000001D1"
-"0000000100000000000001D2"
-"0000000100000000000001D3"
-"0000000100000000000001D4"
-"0000000100000000000001D5"
-"0000000100000000000001D6"
-"0000000100000000000001D7"
-"0000000100000000000001D8"
-"0000000100000000000001D9"
-"0000000100000000000001DA"
-"0000000100000000000001DB"
-"0000000100000000000001DC"
-"0000000100000000000001DD"
-"0000000100000000000001DE"
-"0000000100000000000001DF"
-"0000000100000000000001E0"
-"0000000100000000000001E1"
-"0000000100000000000001E2"
-"0000000100000000000001E3"
-"0000000100000000000001E4"
-"0000000100000000000001E5"
-"0000000100000000000001E6"
-"0000000100000000000001E7"
-"0000000100000000000001E8"
-"0000000100000000000001E9"
-"0000000100000000000001EA"
-"0000000100000000000001EB"
-"0000000100000000000001EC"
-"0000000100000000000001ED"
-"0000000100000000000001EE"
-"0000000100000000000001EF"
-"0000000100000000000001F0"
-"0000000100000000000001F1"
-"0000000100000000000001F2"
-"0000000100000000000001F3"
-"0000000100000000000001F4"
-"0000000100000000000001F5"
-"0000000100000000000001F6"
-"0000000100000000000001F7"
-"0000000100000000000001F8"
-"0000000100000000000001F9"
-"0000000100000000000001FA"
-"0000000100000000000001FB"
-"0000000100000000000001FC"
-"0000000100000000000001FD"
-"0000000100000000000001FE"
-"0000000100000000000001FF"
-"000000010000000000000200"
-"000000010000000000000201"
-"000000010000000000000202"
-"000000010000000000000203"
-"000000010000000000000204"
-"000000010000000000000205"
-"000000010000000000000206"
-"000000010000000000000207"
-"000000010000000000000208"
-"000000010000000000000209"
-"00000001000000000000020A"
-"00000001000000000000020B"
-"00000001000000000000020C"
-"00000001000000000000020D"
-"00000001000000000000020E"
-"00000001000000000000020F"
-"000000010000000000000210"
-"000000010000000000000211"
-"000000010000000000000212"
-"000000010000000000000213"
-"000000010000000000000214"
-"000000010000000000000215"
-"000000010000000000000216"
-"000000010000000000000217"
-"000000010000000000000218"
-"000000010000000000000219"
-"00000001000000000000021A"
-"00000001000000000000021B"
-"00000001000000000000021C"
-"00000001000000000000021D"
-"00000001000000000000021E"
-"00000001000000000000021F"
-"000000010000000000000220"
-"000000010000000000000221"
-"000000010000000000000222"
-"000000010000000000000223"
-"000000010000000000000224"
-"000000010000000000000225"
-"000000010000000000000226"
-"000000010000000000000227"
-"000000010000000000000228"
-"000000010000000000000229"
-"00000001000000000000022A"
-"00000001000000000000022B"
-"00000001000000000000022C"
-"00000001000000000000022D"
-"00000001000000000000022E"
-"00000001000000000000022F"
-"000000010000000000000230"
-"000000010000000000000231"
-"000000010000000000000232"
-"000000010000000000000233"
-"000000010000000000000234"
-"000000010000000000000235"
-"000000010000000000000236"
-"000000010000000000000237"
-"000000010000000000000238"
-"000000010000000000000239"
-"00000001000000000000023A"
-"00000001000000000000023B"
-"00000001000000000000023C"
-"00000001000000000000023D"
-"00000001000000000000023E"
-"00000001000000000000023F"
-"000000010000000000000240"
-"000000010000000000000241"
-"000000010000000000000242"
-"000000010000000000000243"
-"000000010000000000000244"
-"000000010000000000000245"
-"000000010000000000000246"
-"000000010000000000000247"
-"000000010000000000000248"
-"000000010000000000000249"
-"00000001000000000000024A"
-"00000001000000000000024B"
-"00000001000000000000024C"
-"00000001000000000000024D"
-"00000001000000000000024E"
-"00000001000000000000024F"
-"000000010000000000000250"
-"000000010000000000000251"
-"000000010000000000000252"
-"000000010000000000000253"
-"000000010000000000000254"
-"000000010000000000000255"
-"000000010000000000000256"
-"000000010000000000000257"
-"000000010000000000000258"
-"000000010000000000000259"
-"00000001000000000000025A"
-"00000001000000000000025B"
-"00000001000000000000025C"
-"00000001000000000000025D"
-"00000001000000000000025E"
-"00000001000000000000025F"
-"000000010000000000000260"
-"000000010000000000000261"
-"000000010000000000000262"
-"000000010000000000000263"
-"000000010000000000000264"
-"000000010000000000000265"
-"000000010000000000000266"
-"000000010000000000000267"
-"000000010000000000000268"
-"000000010000000000000269"
-"00000001000000000000026A"
-"00000001000000000000026B"
-"00000001000000000000026C"
-"00000001000000000000026D"
-"00000001000000000000026E"
-"00000001000000000000026F"
-"000000010000000000000270"
-"000000010000000000000271"
-"000000010000000000000272"
-"000000010000000000000273"
-"000000010000000000000274"
-"000000010000000000000275"
-"000000010000000000000276"
-"000000010000000000000277"
-"000000010000000000000278"
-"000000010000000000000279"
-"00000001000000000000027A"
-"00000001000000000000027B"
-"00000001000000000000027C"
-"00000001000000000000027D"
-"00000001000000000000027E"
-"00000001000000000000027F"
-"000000010000000000000280"
-"000000010000000000000281"
-"000000010000000000000282"
-"000000010000000000000283"
-"000000010000000000000284"
-"000000010000000000000285"
-"000000010000000000000286"
-"000000010000000000000287"
-"000000010000000000000288"
-"000000010000000000000289"
-"00000001000000000000028A"
-"00000001000000000000028B"
-"00000001000000000000028C"
-"00000001000000000000028D"
-"00000001000000000000028E"
-"00000001000000000000028F"
-"000000010000000000000290"
-"000000010000000000000291"
-"000000010000000000000292"
-"000000010000000000000293"
-"000000010000000000000294"
-"000000010000000000000295"
-"000000010000000000000296"
-"000000010000000000000297"
-"000000010000000000000298"
-"000000010000000000000299"
-"00000001000000000000029A"
-"00000001000000000000029B"
-"00000001000000000000029C"
-"00000001000000000000029D"
-"00000001000000000000029E"
-"00000001000000000000029F"
-"0000000100000000000002A0"
-"0000000100000000000002A1"
-"0000000100000000000002A2"
-"0000000100000000000002A3"
-"0000000100000000000002A4"
-"0000000100000000000002A5"
-"0000000100000000000002A6"
-"0000000100000000000002A7"
-"0000000100000000000002A8"
-"0000000100000000000002A9"
-"0000000100000000000002AA"
-"0000000100000000000002AB"
-"0000000100000000000002AC"
-"0000000100000000000002AD"
-"0000000100000000000002AE"
-"0000000100000000000002AF"
-"0000000100000000000002B0"
-"0000000100000000000002B1"
-"0000000100000000000002B2"
-"0000000100000000000002B3"
-"0000000100000000000002B4"
-"0000000100000000000002B5"
-"0000000100000000000002B6"
-"0000000100000000000002B7"
-"0000000100000000000002B8"
-"0000000100000000000002B9"
-"0000000100000000000002BA"
-"0000000100000000000002BB"
-"0000000100000000000002BC"
-"0000000100000000000002BD"
-"0000000100000000000002BE"
-"0000000100000000000002BF"
-"0000000100000000000002C0"
-"0000000100000000000002C1"
-"0000000100000000000002C2"
-"0000000100000000000002C3"
-"0000000100000000000002C4"
-"0000000100000000000002C5"
-"0000000100000000000002C6"
-"0000000100000000000002C7"
-"0000000100000000000002C8"
-"0000000100000000000002C9"
-"0000000100000000000002CA"
-"0000000100000000000002CB"
-"0000000100000000000002CC"
-"0000000100000000000002CD"
-"0000000100000000000002CE"
-"0000000100000000000002CF"
-"0000000100000000000002D0"
-"0000000100000000000002D1"
-"0000000100000000000002D2"
-"0000000100000000000002D3"
-"0000000100000000000002D4"
-"0000000100000000000002D5"
-"0000000100000000000002D6"
-"0000000100000000000002D7"
-"0000000100000000000002D8"
-"0000000100000000000002D9"
-"0000000100000000000002DA"
-"0000000100000000000002DB"
-"0000000100000000000002DC"
-"0000000100000000000002DD"
-"0000000100000000000002DE"
-"0000000100000000000002DF"
-"0000000100000000000002E0"
-"0000000100000000000002E1"
-"0000000100000000000002E2"
-"0000000100000000000002E3"
-"0000000100000000000002E4"
-"0000000100000000000002E5"
-"0000000100000000000002E6"
-"0000000100000000000002E7"
-"0000000100000000000002E8"
-"0000000100000000000002E9"
-"0000000100000000000002EA"
-"0000000100000000000002EB"
-"0000000100000000000002EC"
-"0000000100000000000002ED"
-"0000000100000000000002EE"
-"0000000100000000000002EF"
-"0000000100000000000002F0"
-"0000000100000000000002F1"
-"0000000100000000000002F2"
-"0000000100000000000002F3"
-"0000000100000000000002F4"
-"0000000100000000000002F5"
-"0000000100000000000002F6"
-"0000000100000000000002F7"
-"0000000100000000000002F8"
-"0000000100000000000002F9"
-"0000000100000000000002FA"
-"0000000100000000000002FB"
-"0000000100000000000002FC"
-"0000000100000000000002FD"
-"0000000100000000000002FE"
-"0000000100000000000002FF"
-"000000010000000000000300"
-"000000010000000000000301"
-"000000010000000000000302"
-"000000010000000000000303"
-"000000010000000000000304"
-"000000010000000000000305"
-"000000010000000000000306"
-"000000010000000000000307"
-"000000010000000000000308"
-"000000010000000000000309"
-"00000001000000000000030A"
-"00000001000000000000030B"
-"00000001000000000000030C"
-"00000001000000000000030D"
-"00000001000000000000030E"
-"00000001000000000000030F"
-"000000010000000000000310"
-"000000010000000000000311"
-"000000010000000000000312"
-"000000010000000000000313"
-"000000010000000000000314"
-"000000010000000000000315"
-"000000010000000000000316"
-"000000010000000000000317"
-"000000010000000000000318"
-"000000010000000000000319"
-"00000001000000000000031A"
-"00000001000000000000031B"
-"00000001000000000000031C"
-"00000001000000000000031D"
-"00000001000000000000031E"
-"00000001000000000000031F"
-"000000010000000000000320"
-"000000010000000000000321"
-"000000010000000000000322"
-"000000010000000000000323"
-"000000010000000000000324"
-"000000010000000000000325"
-"000000010000000000000326"
-"000000010000000000000327"
-"000000010000000000000328"
-"000000010000000000000329"
-"00000001000000000000032A"
-"00000001000000000000032B"
-"00000001000000000000032C"
-"00000001000000000000032D"
-"00000001000000000000032E"
-"00000001000000000000032F"
-"000000010000000000000330"
-"000000010000000000000331"
-"000000010000000000000332"
-"000000010000000000000333"
-"000000010000000000000334"
-"000000010000000000000335"
-"000000010000000000000336"
-"000000010000000000000337"
-"000000010000000000000338"
-"000000010000000000000339"
-"00000001000000000000033A"
-"00000001000000000000033B"
-"00000001000000000000033C"
-"00000001000000000000033D"
-"00000001000000000000033E"
-"00000001000000000000033F"
-"000000010000000000000340"
-"000000010000000000000341"
-"000000010000000000000342"
-"000000010000000000000343"
-"000000010000000000000344"
-"000000010000000000000345"
-"000000010000000000000346"
-"000000010000000000000347"
-"000000010000000000000348"
-"000000010000000000000349"
-"00000001000000000000034A"
-"00000001000000000000034B"
-"00000001000000000000034C"
-"00000001000000000000034D"
-"00000001000000000000034E"
-"00000001000000000000034F"
-"000000010000000000000350"
-"000000010000000000000351"
-"000000010000000000000352"
-"000000010000000000000353"
-"000000010000000000000354"
-"000000010000000000000355"
-"000000010000000000000356"
-"000000010000000000000357"
-"000000010000000000000358"
-"000000010000000000000359"
-"00000001000000000000035A"
-"00000001000000000000035B"
-"00000001000000000000035C"
-"00000001000000000000035D"
-"00000001000000000000035E"
-"00000001000000000000035F"
-"000000010000000000000360"
-"000000010000000000000361"
-"000000010000000000000362"
-"000000010000000000000363"
-"000000010000000000000364"
-"000000010000000000000365"
-"000000010000000000000366"
-"000000010000000000000367"
-"000000010000000000000368"
-"000000010000000000000369"
-"00000001000000000000036A"
-"00000001000000000000036B"
-"00000001000000000000036C"
-"00000001000000000000036D"
-"00000001000000000000036E"
-"00000001000000000000036F"
-"000000010000000000000370"
-"000000010000000000000371"
-"000000010000000000000372"
-"000000010000000000000373"
-"000000010000000000000374"
-"000000010000000000000375"
-"000000010000000000000376"
-"000000010000000000000377"
-"000000010000000000000378"
-"000000010000000000000379"
-"00000001000000000000037A"
-"00000001000000000000037B"
-"00000001000000000000037C"
-"00000001000000000000037D"
-"00000001000000000000037E"
-"00000001000000000000037F"
-"000000010000000000000380"
-"000000010000000000000381"
-"000000010000000000000382"
-"000000010000000000000383"
-"000000010000000000000384"
-"000000010000000000000385"
-"000000010000000000000386"
-"000000010000000000000387"
-"000000010000000000000388"
-"000000010000000000000389"
-"00000001000000000000038A"
-"00000001000000000000038B"
-"00000001000000000000038C"
-"00000001000000000000038D"
-"00000001000000000000038E"
-"00000001000000000000038F"
-"000000010000000000000390"
-"000000010000000000000391"
-"000000010000000000000392"
-"000000010000000000000393"
-"000000010000000000000394"
-"000000010000000000000395"
-"000000010000000000000396"
-"000000010000000000000397"
-"000000010000000000000398"
-"000000010000000000000399"
-"00000001000000000000039A"
-"00000001000000000000039B"
-"00000001000000000000039C"
-"00000001000000000000039D"
-"00000001000000000000039E"
-"00000001000000000000039F"
-"0000000100000000000003A0"
-"0000000100000000000003A1"
-"0000000100000000000003A2"
-"0000000100000000000003A3"
-"0000000100000000000003A4"
-"0000000100000000000003A5"
-"0000000100000000000003A6"
-"0000000100000000000003A7"
-"0000000100000000000003A8"
-"0000000100000000000003A9"
-"0000000100000000000003AA"
-"0000000100000000000003AB"
-"0000000100000000000003AC"
-"0000000100000000000003AD"
-"0000000100000000000003AE"
-"0000000100000000000003AF"
-"0000000100000000000003B0"
-"0000000100000000000003B1"
-"0000000100000000000003B2"
-"0000000100000000000003B3"
-"0000000100000000000003B4"
-"0000000100000000000003B5"
-"0000000100000000000003B6"
-"0000000100000000000003B7"
-"0000000100000000000003B8"
-"0000000100000000000003B9"
-"0000000100000000000003BA"
-"0000000100000000000003BB"
-"0000000100000000000003BC"
-"0000000100000000000003BD"
-"0000000100000000000003BE"
-"0000000100000000000003BF"
-"0000000100000000000003C0"
-"0000000100000000000003C1"
-"0000000100000000000003C2"
-"0000000100000000000003C3"
-"0000000100000000000003C4"
-"0000000100000000000003C5"
-"0000000100000000000003C6"
-"0000000100000000000003C7"
-"0000000100000000000003C8"
-"0000000100000000000003C9"
-"0000000100000000000003CA"
-"0000000100000000000003CB"
-"0000000100000000000003CC"
-"0000000100000000000003CD"
-"0000000100000000000003CE"
-"0000000100000000000003CF"
-"0000000100000000000003D0"
-"0000000100000000000003D1"
-"0000000100000000000003D2"
-"0000000100000000000003D3"
-"0000000100000000000003D4"
-"0000000100000000000003D5"
-"0000000100000000000003D6"
-"0000000100000000000003D7"
-"0000000100000000000003D8"
-"0000000100000000000003D9"
-"0000000100000000000003DA"
-"0000000100000000000003DB"
-"0000000100000000000003DC"
-"0000000100000000000003DD"
-"0000000100000000000003DE"
-"0000000100000000000003DF"
-"0000000100000000000003E0"
-"0000000100000000000003E1"
-"0000000100000000000003E2"
-"0000000100000000000003E3"
-"0000000100000000000003E4"
-"0000000100000000000003E5"
-"0000000100000000000003E6"
-"0000000100000000000003E7"
-"0000000100000000000003E8"
-"0000000100000000000003E9"
-"0000000100000000000003EA"
-"0000000100000000000003EB"
-"0000000100000000000003EC"
-"0000000100000000000003ED"
-"0000000100000000000003EE"
-"0000000100000000000003EF"
-"0000000100000000000003F0"
-"0000000100000000000003F1"
-"0000000100000000000003F2"
-"0000000100000000000003F3"
-"0000000100000000000003F4"
-"0000000100000000000003F5"
-"0000000100000000000003F6"
-"0000000100000000000003F7"
-"0000000100000000000003F8"
-"0000000100000000000003F9"
-"0000000100000000000003FA"
-"0000000100000000000003FB"
-"0000000100000000000003FC"
-"0000000100000000000003FD"
-"0000000100000000000003FE"
-"0000000100000000000003FF"
-"000000010000000000000400"
-"000000010000000000000401"
-"000000010000000000000402"
-"000000010000000000000403"
-"000000010000000000000404"
-"000000010000000000000405"
-"000000010000000000000406"
-"000000010000000000000407"
-"000000010000000000000408"
-"000000010000000000000409"
-"00000001000000000000040A"
-"00000001000000000000040B"
-"00000001000000000000040C"
-"00000001000000000000040D"
-"00000001000000000000040E"
-"00000001000000000000040F"
-"000000010000000000000410"
-"000000010000000000000411"
-"000000010000000000000412"
-"000000010000000000000413"
-"000000010000000000000414"
-"000000010000000000000415"
-"000000010000000000000416"
-"000000010000000000000417"
-"000000010000000000000418"
-"000000010000000000000419"
-"00000001000000000000041A"
-"00000001000000000000041B"
-"00000001000000000000041C"
-"00000001000000000000041D"
-"00000001000000000000041E"
-"00000001000000000000041F"
-"000000010000000000000420"
-"000000010000000000000421"
-"000000010000000000000422"
-"000000010000000000000423"
-"000000010000000000000424"
-"000000010000000000000425"
-"000000010000000000000426"
-"000000010000000000000427"
-"000000010000000000000428"
-"000000010000000000000429"
-"00000001000000000000042A"
-"00000001000000000000042B"
-"00000001000000000000042C"
-"00000001000000000000042D"
-"00000001000000000000042E"
-"00000001000000000000042F"
-"000000010000000000000430"
-"000000010000000000000431"
-"000000010000000000000432"
-"000000010000000000000433"
-"000000010000000000000434"
-"000000010000000000000435"
-"000000010000000000000436"
-"000000010000000000000437"
-"000000010000000000000438"
-"000000010000000000000439"
-"00000001000000000000043A"
-"00000001000000000000043B"
-"00000001000000000000043C"
-"00000001000000000000043D"
-"00000001000000000000043E"
-"00000001000000000000043F"
-"000000010000000000000440"
-"000000010000000000000441"
-"000000010000000000000442"
-"000000010000000000000443"
-"000000010000000000000444"
-"000000010000000000000445"
-"000000010000000000000446"
-"000000010000000000000447"
-"000000010000000000000448"
-"000000010000000000000449"
-"00000001000000000000044A"
-"00000001000000000000044B"
-"00000001000000000000044C"
-"00000001000000000000044D"
-"00000001000000000000044E"
-"00000001000000000000044F"
-"000000010000000000000450"
-"000000010000000000000451"
-"000000010000000000000452"
-"000000010000000000000453"
-"000000010000000000000454"
-"000000010000000000000455"
-"000000010000000000000456"
-"000000010000000000000457"
-"000000010000000000000458"
-"000000010000000000000459"
-"00000001000000000000045A"
-"00000001000000000000045B"
-"00000001000000000000045C"
-"00000001000000000000045D"
-"00000001000000000000045E"
-"00000001000000000000045F"
-"000000010000000000000460"
-"000000010000000000000461"
-"000000010000000000000462"
-"000000010000000000000463"
-"000000010000000000000464"
-"000000010000000000000465"
-"000000010000000000000466"
-"000000010000000000000467"
-"000000010000000000000468"
-"000000010000000000000469"
-"00000001000000000000046A"
-"00000001000000000000046B"
-"00000001000000000000046C"
-"00000001000000000000046D"
-"00000001000000000000046E"
-"00000001000000000000046F"
-"000000010000000000000470"
-"000000010000000000000471"
-"000000010000000000000472"
-"000000010000000000000473"
-"000000010000000000000474"
-"000000010000000000000475"
-"000000010000000000000476"
-"000000010000000000000477"
-"000000010000000000000478"
-"000000010000000000000479"
-"00000001000000000000047A"
-"00000001000000000000047B"
-"00000001000000000000047C"
-"00000001000000000000047D"
-"00000001000000000000047E"
-"00000001000000000000047F"
-"000000010000000000000480"
-"000000010000000000000481"
-"000000010000000000000482"
-"000000010000000000000483"
-"000000010000000000000484"
-"000000010000000000000485"
-"000000010000000000000486"
-"000000010000000000000487"
-"000000010000000000000488"
-"000000010000000000000489"
-"00000001000000000000048A"
-"00000001000000000000048B"
-"00000001000000000000048C"
-"00000001000000000000048D"
-"00000001000000000000048E"
-"00000001000000000000048F"
-"000000010000000000000490"
-"000000010000000000000491"
-"000000010000000000000492"
-"000000010000000000000493"
-"000000010000000000000494"
-"000000010000000000000495"
-"000000010000000000000496"
-"000000010000000000000497"
-"000000010000000000000498"
-"000000010000000000000499"
-"00000001000000000000049A"
-"00000001000000000000049B"
-"00000001000000000000049C"
-"00000001000000000000049D"
-"00000001000000000000049E"
-"00000001000000000000049F"
-"0000000100000000000004A0"
-"0000000100000000000004A1"
-"0000000100000000000004A2"
-"0000000100000000000004A3"
-"0000000100000000000004A4"
-"0000000100000000000004A5"
-"0000000100000000000004A6"
-"0000000100000000000004A7"
-"0000000100000000000004A8"
-"0000000100000000000004A9"
-"0000000100000000000004AA"
-"0000000100000000000004AB"
-"0000000100000000000004AC"
-"0000000100000000000004AD"
-"0000000100000000000004AE"
-"0000000100000000000004AF"
-"0000000100000000000004B0"
-"0000000100000000000004B1"
-"0000000100000000000004B2"
-"0000000100000000000004B3"
-"0000000100000000000004B4"
-"0000000100000000000004B5"
-"0000000100000000000004B6"
-"0000000100000000000004B7"
-"0000000100000000000004B8"
-"0000000100000000000004B9"
-"0000000100000000000004BA"
-"0000000100000000000004BB"
-"0000000100000000000004BC"
-"0000000100000000000004BD"
-"0000000100000000000004BE"
-"0000000100000000000004BF"
-"0000000100000000000004C0"
-"0000000100000000000004C1"
-"0000000100000000000004C2"
-"0000000100000000000004C3"
-"0000000100000000000004C4"
-"0000000100000000000004C5"
-"0000000100000000000004C6"
-"0000000100000000000004C7"
-"0000000100000000000004C8"
-"0000000100000000000004C9"
-"0000000100000000000004CA"
-"0000000100000000000004CB"
-"0000000100000000000004CC"
-"0000000100000000000004CD"
-"0000000100000000000004CE"
-"0000000100000000000004CF"
-"0000000100000000000004D0"
-"0000000100000000000004D1"
-"0000000100000000000004D2"
-"0000000100000000000004D3"
-"0000000100000000000004D4"
-"0000000100000000000004D5"
-"0000000100000000000004D6"
-"0000000100000000000004D7"
-"0000000100000000000004D8"
-"0000000100000000000004D9"
-"0000000100000000000004DA"
-"0000000100000000000004DB"
-"0000000100000000000004DC"
-"0000000100000000000004DD"
-"0000000100000000000004DE"
-"0000000100000000000004DF"
-"0000000100000000000004E0"
-"0000000100000000000004E1"
-"0000000100000000000004E2"
-"0000000100000000000004E3"
-"0000000100000000000004E4"
-"0000000100000000000004E5"
-"0000000100000000000004E6"
-"0000000100000000000004E7"
-"0000000100000000000004E8"
-"0000000100000000000004E9"
-"0000000100000000000004EA"
-"0000000100000000000004EB"
-"0000000100000000000004EC"
-"0000000100000000000004ED"
-"0000000100000000000004EE"
-"0000000100000000000004EF"
-"0000000100000000000004F0"
-"0000000100000000000004F1"
-"0000000100000000000004F2"
-"0000000100000000000004F3"
-"0000000100000000000004F4"
-"0000000100000000000004F5"
-"0000000100000000000004F6"
-"0000000100000000000004F7"
-"0000000100000000000004F8"
-"0000000100000000000004F9"
-"0000000100000000000004FA"
-"0000000100000000000004FB"
-"0000000100000000000004FC"
-"0000000100000000000004FD"
-"0000000100000000000004FE"
-"0000000100000000000004FF"
-"000000010000000000000500"
-"000000010000000000000501"
-"000000010000000000000502"
-"000000010000000000000503"
-"000000010000000000000504"
-"000000010000000000000505"
-"000000010000000000000506"
-"000000010000000000000507"
-"000000010000000000000508"
-"000000010000000000000509"
-"00000001000000000000050A"
-"00000001000000000000050B"
-"00000001000000000000050C"
-"00000001000000000000050D"
-"00000001000000000000050E"
-"00000001000000000000050F"
-"000000010000000000000510"
-"000000010000000000000511"
-"000000010000000000000512"
-"000000010000000000000513"
-"000000010000000000000514"
-"000000010000000000000515"
-"000000010000000000000516"
-"000000010000000000000517"
-"000000010000000000000518"
-"000000010000000000000519"
-"00000001000000000000051A"
-"00000001000000000000051B"
-"00000001000000000000051C"
-"00000001000000000000051D"
-"00000001000000000000051E"
-"00000001000000000000051F"
-"000000010000000000000520"
-"000000010000000000000521"
-"000000010000000000000522"
-"000000010000000000000523"
-"000000010000000000000524"
-"000000010000000000000525"
-"000000010000000000000526"
-"000000010000000000000527"
-"000000010000000000000528"
-"000000010000000000000529"
-"00000001000000000000052A"
-"00000001000000000000052B"
-"00000001000000000000052C"
-"00000001000000000000052D"
-"00000001000000000000052E"
-"00000001000000000000052F"
-"000000010000000000000530"
-"000000010000000000000531"
-"000000010000000000000532"
-"000000010000000000000533"
-"000000010000000000000534"
-"000000010000000000000535"
-"000000010000000000000536"
-"000000010000000000000537"
-"000000010000000000000538"
-"000000010000000000000539"
-"00000001000000000000053A"
-"00000001000000000000053B"
-"00000001000000000000053C"
-"00000001000000000000053D"
-"00000001000000000000053E"
-"00000001000000000000053F"
-"000000010000000000000540"
-"000000010000000000000541"
-"000000010000000000000542"
-"000000010000000000000543"
-"000000010000000000000544"
-"000000010000000000000545"
-"000000010000000000000546"
-"000000010000000000000547"
-"000000010000000000000548"
-"000000010000000000000549"
-"00000001000000000000054A"
-"00000001000000000000054B"
-"00000001000000000000054C"
-"00000001000000000000054D"
-"00000001000000000000054E"
-"00000001000000000000054F"
-"000000010000000000000550"
-"000000010000000000000551"
-"000000010000000000000552"
-"000000010000000000000553"
-"000000010000000000000554"
-"000000010000000000000555"
-"000000010000000000000556"
-"000000010000000000000557"
-"000000010000000000000558"
-"000000010000000000000559"
-"00000001000000000000055A"
-"00000001000000000000055B"
-"00000001000000000000055C"
-"00000001000000000000055D"
-"00000001000000000000055E"
-"00000001000000000000055F"
-"000000010000000000000560"
-"000000010000000000000561"
-"000000010000000000000562"
-"000000010000000000000563"
-"000000010000000000000564"
-"000000010000000000000565"
-"000000010000000000000566"
-"000000010000000000000567"
-"000000010000000000000568"
-"000000010000000000000569"
-"00000001000000000000056A"
-"00000001000000000000056B"
-"00000001000000000000056C"
-"00000001000000000000056D"
-"00000001000000000000056E"
-"00000001000000000000056F"
-"000000010000000000000570"
-"000000010000000000000571"
-"000000010000000000000572"
-"000000010000000000000573"
-"000000010000000000000574"
-"000000010000000000000575"
-"000000010000000000000576"
-"000000010000000000000577"
-"000000010000000000000578"
-"000000010000000000000579"
-"00000001000000000000057A"
-"00000001000000000000057B"
-"00000001000000000000057C"
-"00000001000000000000057D"
-"00000001000000000000057E"
-"00000001000000000000057F"
-"000000010000000000000580"
-"000000010000000000000581"
-"000000010000000000000582"
-"000000010000000000000583"
-"000000010000000000000584"
-"000000010000000000000585"
-"000000010000000000000586"
-"000000010000000000000587"
-"000000010000000000000588"
-"000000010000000000000589"
-"00000001000000000000058A"
-"00000001000000000000058B"
-"00000001000000000000058C"
-"00000001000000000000058D"
-"00000001000000000000058E"
-"00000001000000000000058F"
-"000000010000000000000590"
-"000000010000000000000591"
-"000000010000000000000592"
-"000000010000000000000593"
-"000000010000000000000594"
-"000000010000000000000595"
-"000000010000000000000596"
-"000000010000000000000597"
-"000000010000000000000598"
-"000000010000000000000599"
-"00000001000000000000059A"
-"00000001000000000000059B"
-"00000001000000000000059C"
-"00000001000000000000059D"
-"00000001000000000000059E"
-"00000001000000000000059F"
-"0000000100000000000005A0"
-"0000000100000000000005A1"
-"0000000100000000000005A2"
-"0000000100000000000005A3"
-"0000000100000000000005A4"
-"0000000100000000000005A5"
-"0000000100000000000005A6"
-"0000000100000000000005A7"
-"0000000100000000000005A8"
-"0000000100000000000005A9"
-"0000000100000000000005AA"
-"0000000100000000000005AB"
-"0000000100000000000005AC"
-"0000000100000000000005AD"
-"0000000100000000000005AE"
-"0000000100000000000005AF"
-"0000000100000000000005B0"
-"0000000100000000000005B1"
-"0000000100000000000005B2"
-"0000000100000000000005B3"
-"0000000100000000000005B4"
-"0000000100000000000005B5"
-"0000000100000000000005B6"
-"0000000100000000000005B7"
-"0000000100000000000005B8"
-"0000000100000000000005B9"
-"0000000100000000000005BA"
-"0000000100000000000005BB"
-"0000000100000000000005BC"
-"0000000100000000000005BD"
-"0000000100000000000005BE"
-"0000000100000000000005BF"
-"0000000100000000000005C0"
-"0000000100000000000005C1"
-"0000000100000000000005C2"
-"0000000100000000000005C3"
-"0000000100000000000005C4"
-"0000000100000000000005C5"
-"0000000100000000000005C6"
-"0000000100000000000005C7"
-"0000000100000000000005C8"
-"0000000100000000000005C9"
-"0000000100000000000005CA"
-"0000000100000000000005CB"
-"0000000100000000000005CC"
-"0000000100000000000005CD"
-"0000000100000000000005CE"
-"0000000100000000000005CF"
-"0000000100000000000005D0"
-"0000000100000000000005D1"
-"0000000100000000000005D2"
-"0000000100000000000005D3"
-"0000000100000000000005D4"
-"0000000100000000000005D5"
-"0000000100000000000005D6"
-"0000000100000000000005D7"
-"0000000100000000000005D8"
-"0000000100000000000005D9"
-"0000000100000000000005DA"
-"0000000100000000000005DB"
-"0000000100000000000005DC"
-"0000000100000000000005DD"
-"0000000100000000000005DE"
-"0000000100000000000005DF"
-"0000000100000000000005E0"
-"0000000100000000000005E1"
-"0000000100000000000005E2"
-"0000000100000000000005E3"
-"0000000100000000000005E4"
-"0000000100000000000005E5"
-"0000000100000000000005E6"
-"0000000100000000000005E7"
-"0000000100000000000005E8"
-"0000000100000000000005E9"
-"0000000100000000000005EA"
-"0000000100000000000005EB"
-"0000000100000000000005EC"
-"0000000100000000000005ED"
-"0000000100000000000005EE"
-"0000000100000000000005EF"
-"0000000100000000000005F0"
-"0000000100000000000005F1"
-"0000000100000000000005F2"
-"0000000100000000000005F3"
-"0000000100000000000005F4"
-"000000020000000000000001"
-"000000020000000000000002"
-"000000020000000000000003"
-"000000020000000000000004"
-"000000020000000000000005"
-"000000020000000000000006"
-"000000020000000000000007"
-"000000020000000000000008"
-"000000020000000000000009"
-"00000002000000000000000A"
-"00000002000000000000000B"
-"00000002000000000000000C"
-"00000002000000000000000D"
-"00000002000000000000000E"
-"00000002000000000000000F"
-"000000020000000000000010"
-"000000020000000000000011"
-"000000020000000000000012"
-"000000020000000000000013"
-"000000020000000000000014"
-"000000020000000000000015"
-"000000020000000000000016"
-"000000020000000000000017"
-"000000020000000000000018"
-"000000020000000000000019"
-"00000002000000000000001A"
-"00000002000000000000001B"
-"00000002000000000000001C"
-"00000002000000000000001D"
-"00000002000000000000001E"
-"00000002000000000000001F"
-"000000020000000000000020"
-"000000020000000000000021"
-"000000020000000000000022"
-"000000020000000000000023"
-"000000020000000000000024"
-"000000020000000000000025"
-"000000020000000000000026"
-"000000020000000000000027"
-"000000020000000000000028"
-"000000020000000000000029"
-"00000002000000000000002A"
-"00000002000000000000002B"
-"00000002000000000000002C"
-"00000002000000000000002D"
-"00000002000000000000002E"
-"00000002000000000000002F"
-"000000020000000000000030"
-"000000020000000000000031"
-"000000020000000000000032"
-"000000020000000000000033"
-"000000020000000000000034"
-"000000020000000000000035"
-"000000020000000000000036"
-"000000020000000000000037"
-"000000020000000000000038"
-"000000020000000000000039"
-"00000002000000000000003A"
-"00000002000000000000003B"
-"00000002000000000000003C"
-"00000002000000000000003D"
-"00000002000000000000003E"
-"00000002000000000000003F"
-"000000020000000000000040"
-"000000020000000000000041"
-"000000020000000000000042"
-"000000020000000000000043"
-"000000020000000000000044"
-"000000020000000000000045"
-"000000020000000000000046"
-"000000020000000000000047"
-"000000020000000000000048"
-"000000020000000000000049"
-"00000002000000000000004A"
-"00000002000000000000004B"
-"00000002000000000000004C"
-"00000002000000000000004D"
-"00000002000000000000004E"
-"00000002000000000000004F"
-"000000020000000000000050"
-"000000020000000000000051"
-"000000020000000000000052"
-"000000020000000000000053"
-"000000020000000000000054"
-"000000020000000000000055"
-"000000020000000000000056"
-"000000020000000000000057"
-"000000020000000000000058"
-"000000020000000000000059"
-"00000002000000000000005A"
-"00000002000000000000005B"
-"00000002000000000000005C"
-"00000002000000000000005D"
-"00000002000000000000005E"
-"00000002000000000000005F"
-"000000020000000000000060"
-"000000020000000000000061"
-"000000020000000000000062"
-"000000020000000000000063"
-"000000020000000000000064"
-"000000020000000000000065"
-"000000020000000000000066"
-"000000020000000000000067"
-"000000020000000000000068"
-"000000020000000000000069"
-"00000002000000000000006A"
-"00000002000000000000006B"
-"00000002000000000000006C"
-"00000002000000000000006D"
-"00000002000000000000006E"
-"00000002000000000000006F"
-"000000020000000000000070"
-"000000020000000000000071"
-"000000020000000000000072"
-"000000020000000000000073"
-"000000020000000000000074"
-"000000020000000000000075"
-"000000020000000000000076"
-"000000020000000000000077"
-"000000020000000000000078"
-"000000020000000000000079"
-"00000002000000000000007A"
-"00000002000000000000007B"
-"00000002000000000000007C"
-"00000002000000000000007D"
-"00000002000000000000007E"
-"00000002000000000000007F"
-"000000020000000000000080"
-"000000020000000000000081"
-"000000020000000000000082"
-"000000020000000000000083"
-"000000020000000000000084"
-"000000020000000000000085"
-"000000020000000000000086"
-"000000020000000000000087"
-"000000020000000000000088"
-"000000020000000000000089"
-"00000002000000000000008A"
-"00000002000000000000008B"
-"00000002000000000000008C"
-"00000002000000000000008D"
-"00000002000000000000008E"
-"00000002000000000000008F"
-"000000020000000000000090"
-"000000020000000000000091"
-"000000020000000000000092"
-"000000020000000000000093"
-"000000020000000000000094"
-"000000020000000000000095"
-"000000020000000000000096"
-"000000020000000000000097"
-"000000020000000000000098"
-"000000020000000000000099"
-"00000002000000000000009A"
-"00000002000000000000009B"
-"00000002000000000000009C"
-"00000002000000000000009D"
-"00000002000000000000009E"
-"00000002000000000000009F"
-"0000000200000000000000A0"
-"0000000200000000000000A1"
-"0000000200000000000000A2"
-"0000000200000000000000A3"
-"0000000200000000000000A4"
-"0000000200000000000000A5"
-"0000000200000000000000A6"
-"0000000200000000000000A7"
-"0000000200000000000000A8"
-"0000000200000000000000A9"
-"0000000200000000000000AA"
-"0000000200000000000000AB"
-"0000000200000000000000AC"
-"0000000200000000000000AD"
-"0000000200000000000000AE"
-"0000000200000000000000AF"
-"0000000200000000000000B0"
-"0000000200000000000000B1"
-"0000000200000000000000B2"
-"0000000200000000000000B3"
-"0000000200000000000000B4"
-"0000000200000000000000B5"
-"0000000200000000000000B6"
-"0000000200000000000000B7"
-"0000000200000000000000B8"
-"0000000200000000000000B9"
-"0000000200000000000000BA"
-"0000000200000000000000BB"
-"0000000200000000000000BC"
-"0000000200000000000000BD"
-"0000000200000000000000BE"
-"0000000200000000000000BF"
-"0000000200000000000000C0"
-"0000000200000000000000C1"
-"0000000200000000000000C2"
-"0000000200000000000000C3"
-"0000000200000000000000C4"
-"0000000200000000000000C5"
-"0000000200000000000000C6"
-"0000000200000000000000C7"
-"0000000200000000000000C8"
-"0000000200000000000000C9"
-"0000000200000000000000CA"
-"0000000200000000000000CB"
-"0000000200000000000000CC"
-"0000000200000000000000CD"
-"0000000200000000000000CE"
-"0000000200000000000000CF"
-"0000000200000000000000D0"
-"0000000200000000000000D1"
-"0000000200000000000000D2"
-"0000000200000000000000D3"
-"0000000200000000000000D4"
-"0000000200000000000000D5"
-"0000000200000000000000D6"
-"0000000200000000000000D7"
-"0000000200000000000000D8"
-"0000000200000000000000D9"
-"0000000200000000000000DA"
-"0000000200000000000000DB"
-"0000000200000000000000DC"
-"0000000200000000000000DD"
-"0000000200000000000000DE"
-"0000000200000000000000DF"
-"0000000200000000000000E0"
-"0000000200000000000000E1"
-"0000000200000000000000E2"
-"0000000200000000000000E3"
-"0000000200000000000000E4"
-"0000000200000000000000E5"
-"0000000200000000000000E6"
-"0000000200000000000000E7"
-"0000000200000000000000E8"
-"0000000200000000000000E9"
-"0000000200000000000000EA"
-"0000000200000000000000EB"
-"0000000200000000000000EC"
-"0000000200000000000000ED"
-"0000000200000000000000EE"
-"0000000200000000000000EF"
-"0000000200000000000000F0"
-"0000000200000000000000F1"
-"0000000200000000000000F2"
-"0000000200000000000000F3"
-"0000000200000000000000F4"
-"0000000200000000000000F5"
-"0000000200000000000000F6"
-"0000000200000000000000F7"
-"0000000200000000000000F8"
-"0000000200000000000000F9"
-"0000000200000000000000FA"
-"0000000200000000000000FB"
-"0000000200000000000000FC"
-"0000000200000000000000FD"
-"0000000200000000000000FE"
-"0000000200000000000000FF"
-"000000020000000000000100"
-"000000020000000000000101"
-"000000020000000000000102"
-"000000020000000000000103"
-"000000020000000000000104"
-"000000020000000000000105"
-"000000020000000000000106"
-"000000020000000000000107"
-"000000020000000000000108"
-"000000020000000000000109"
-"00000002000000000000010A"
-"00000002000000000000010B"
-"00000002000000000000010C"
-"00000002000000000000010D"
-"00000002000000000000010E"
-"00000002000000000000010F"
-"000000020000000000000110"
-"000000020000000000000111"
-"000000020000000000000112"
-"000000020000000000000113"
-"000000020000000000000114"
-"000000020000000000000115"
-"000000020000000000000116"
-"000000020000000000000117"
-"000000020000000000000118"
-"000000020000000000000119"
-"00000002000000000000011A"
-"00000002000000000000011B"
-"00000002000000000000011C"
-"00000002000000000000011D"
-"00000002000000000000011E"
-"00000002000000000000011F"
-"000000020000000000000120"
-"000000020000000000000121"
-"000000020000000000000122"
-"000000020000000000000123"
-"000000020000000000000124"
-"000000020000000000000125"
-"000000020000000000000126"
-"000000020000000000000127"
-"000000020000000000000128"
-"000000020000000000000129"
-"00000002000000000000012A"
-"00000002000000000000012B"
-"00000002000000000000012C"
-"00000002000000000000012D"
-"00000002000000000000012E"
-"00000002000000000000012F"
-"000000020000000000000130"
-"000000020000000000000131"
-"000000020000000000000132"
-"000000020000000000000133"
-"000000020000000000000134"
-"000000020000000000000135"
-"000000020000000000000136"
-"000000020000000000000137"
-"000000020000000000000138"
-"000000020000000000000139"
-"00000002000000000000013A"
-"00000002000000000000013B"
-"00000002000000000000013C"
-"00000002000000000000013D"
-"00000002000000000000013E"
-"00000002000000000000013F"
-"000000020000000000000140"
-"000000020000000000000141"
-"000000020000000000000142"
-"000000020000000000000143"
-"000000020000000000000144"
-"000000020000000000000145"
-"000000020000000000000146"
-"000000020000000000000147"
-"000000020000000000000148"
-"000000020000000000000149"
-"00000002000000000000014A"
-"00000002000000000000014B"
-"00000002000000000000014C"
-"00000002000000000000014D"
-"00000002000000000000014E"
-"00000002000000000000014F"
-"000000020000000000000150"
-"000000020000000000000151"
-"000000020000000000000152"
-"000000020000000000000153"
-"000000020000000000000154"
-"000000020000000000000155"
-"000000020000000000000156"
-"000000020000000000000157"
-"000000020000000000000158"
-"000000020000000000000159"
-"00000002000000000000015A"
-"00000002000000000000015B"
-"00000002000000000000015C"
-"00000002000000000000015D"
-"00000002000000000000015E"
-"00000002000000000000015F"
-"000000020000000000000160"
-"000000020000000000000161"
-"000000020000000000000162"
-"000000020000000000000163"
-"000000020000000000000164"
-"000000020000000000000165"
-"000000020000000000000166"
-"000000020000000000000167"
-"000000020000000000000168"
-"000000020000000000000169"
-"00000002000000000000016A"
-"00000002000000000000016B"
-"00000002000000000000016C"
-"00000002000000000000016D"
-"00000002000000000000016E"
-"00000002000000000000016F"
-"000000020000000000000170"
-"000000020000000000000171"
-"000000020000000000000172"
-"000000020000000000000173"
-"000000020000000000000174"
-"000000020000000000000175"
-"000000020000000000000176"
-"000000020000000000000177"
-"000000020000000000000178"
-"000000020000000000000179"
-"00000002000000000000017A"
-"00000002000000000000017B"
-"00000002000000000000017C"
-"00000002000000000000017D"
-"00000002000000000000017E"
-"00000002000000000000017F"
-"000000020000000000000180"
-"000000020000000000000181"
-"000000020000000000000182"
-"000000020000000000000183"
-"000000020000000000000184"
-"000000020000000000000185"
-"000000020000000000000186"
-"000000020000000000000187"
-"000000020000000000000188"
-"000000020000000000000189"
-"00000002000000000000018A"
-"00000002000000000000018B"
-"00000002000000000000018C"
-"00000002000000000000018D"
-"00000002000000000000018E"
-"00000002000000000000018F"
-"000000020000000000000190"
-"000000020000000000000191"
-"000000020000000000000192"
-"000000020000000000000193"
-"000000020000000000000194"
-"000000020000000000000195"
-"000000020000000000000196"
-"000000020000000000000197"
-"000000020000000000000198"
-"000000020000000000000199"
-"00000002000000000000019A"
-"00000002000000000000019B"
-"00000002000000000000019C"
-"00000002000000000000019D"
-"00000002000000000000019E"
-"00000002000000000000019F"
-"0000000200000000000001A0"
-"0000000200000000000001A1"
-"0000000200000000000001A2"
-"0000000200000000000001A3"
-"0000000200000000000001A4"
-"0000000200000000000001A5"
-"0000000200000000000001A6"
-"0000000200000000000001A7"
-"0000000200000000000001A8"
-"0000000200000000000001A9"
-"0000000200000000000001AA"
-"0000000200000000000001AB"
-"0000000200000000000001AC"
-"0000000200000000000001AD"
-"0000000200000000000001AE"
-"0000000200000000000001AF"
-"0000000200000000000001B0"
-"0000000200000000000001B1"
-"0000000200000000000001B2"
-"0000000200000000000001B3"
-"0000000200000000000001B4"
-"0000000200000000000001B5"
-"0000000200000000000001B6"
-"0000000200000000000001B7"
-"0000000200000000000001B8"
-"0000000200000000000001B9"
-"0000000200000000000001BA"
-"0000000200000000000001BB"
-"0000000200000000000001BC"
-"0000000200000000000001BD"
-"0000000200000000000001BE"
-"0000000200000000000001BF"
-"0000000200000000000001C0"
-"0000000200000000000001C1"
-"0000000200000000000001C2"
-"0000000200000000000001C3"
-"0000000200000000000001C4"
-"0000000200000000000001C5"
-"0000000200000000000001C6"
-"0000000200000000000001C7"
-"0000000200000000000001C8"
-"0000000200000000000001C9"
-"0000000200000000000001CA"
-"0000000200000000000001CB"
-"0000000200000000000001CC"
-"0000000200000000000001CD"
-"0000000200000000000001CE"
-"0000000200000000000001CF"
-"0000000200000000000001D0"
-"0000000200000000000001D1"
-"0000000200000000000001D2"
-"0000000200000000000001D3"
-"0000000200000000000001D4"
-"0000000200000000000001D5"
-"0000000200000000000001D6"
-"0000000200000000000001D7"
-"0000000200000000000001D8"
-"0000000200000000000001D9"
-"0000000200000000000001DA"
-"0000000200000000000001DB"
-"0000000200000000000001DC"
-"0000000200000000000001DD"
-"0000000200000000000001DE"
-"0000000200000000000001DF"
-"0000000200000000000001E0"
-"0000000200000000000001E1"
-"0000000200000000000001E2"
-"0000000200000000000001E3"
-"0000000200000000000001E4"
-"0000000200000000000001E5"
-"0000000200000000000001E6"
-"0000000200000000000001E7"
-"0000000200000000000001E8"
-"0000000200000000000001E9"
-"0000000200000000000001EA"
-"0000000200000000000001EB"
-"0000000200000000000001EC"
-"0000000200000000000001ED"
-"0000000200000000000001EE"
-"0000000200000000000001EF"
-"0000000200000000000001F0"
-"0000000200000000000001F1"
-"0000000200000000000001F2"
-"0000000200000000000001F3"
-"0000000200000000000001F4"
-"0000000200000000000001F5"
-"0000000200000000000001F6"
-"0000000200000000000001F7"
-"0000000200000000000001F8"
-"0000000200000000000001F9"
-"0000000200000000000001FA"
-"0000000200000000000001FB"
-"0000000200000000000001FC"
-"0000000200000000000001FD"
-"0000000200000000000001FE"
-"0000000200000000000001FF"
-"000000020000000000000200"
-"000000020000000000000201"
-"000000020000000000000202"
-"000000020000000000000203"
-"000000020000000000000204"
-"000000020000000000000205"
-"000000020000000000000206"
-"000000020000000000000207"
-"000000020000000000000208"
-"000000020000000000000209"
-"00000002000000000000020A"
-"00000002000000000000020B"
-"00000002000000000000020C"
-"00000002000000000000020D"
-"00000002000000000000020E"
-"00000002000000000000020F"
-"000000020000000000000210"
-"000000020000000000000211"
-"000000020000000000000212"
-"000000020000000000000213"
-"000000020000000000000214"
-"000000020000000000000215"
-"000000020000000000000216"
-"000000020000000000000217"
-"000000020000000000000218"
-"000000020000000000000219"
-"00000002000000000000021A"
-"00000002000000000000021B"
-"00000002000000000000021C"
-"00000002000000000000021D"
-"00000002000000000000021E"
-"00000002000000000000021F"
-"000000020000000000000220"
-"000000020000000000000221"
-"000000020000000000000222"
-"000000020000000000000223"
-"000000020000000000000224"
-"000000020000000000000225"
-"000000020000000000000226"
-"000000020000000000000227"
-"000000020000000000000228"
-"000000020000000000000229"
-"00000002000000000000022A"
-"00000002000000000000022B"
-"00000002000000000000022C"
-"00000002000000000000022D"
-"00000002000000000000022E"
-"00000002000000000000022F"
-"000000020000000000000230"
-"000000020000000000000231"
-"000000020000000000000232"
-"000000020000000000000233"
-"000000020000000000000234"
-"000000020000000000000235"
-"000000020000000000000236"
-"000000020000000000000237"
-"000000020000000000000238"
-"000000020000000000000239"
-"00000002000000000000023A"
-"00000002000000000000023B"
-"00000002000000000000023C"
-"00000002000000000000023D"
-"00000002000000000000023E"
-"00000002000000000000023F"
-"000000020000000000000240"
-"000000020000000000000241"
-"000000020000000000000242"
-"000000020000000000000243"
-"000000020000000000000244"
-"000000020000000000000245"
-"000000020000000000000246"
-"000000020000000000000247"
-"000000020000000000000248"
-"000000020000000000000249"
-"00000002000000000000024A"
-"00000002000000000000024B"
-"00000002000000000000024C"
-"00000002000000000000024D"
-"00000002000000000000024E"
-"00000002000000000000024F"
-"000000020000000000000250"
-"000000020000000000000251"
-"000000020000000000000252"
-"000000020000000000000253"
-"000000020000000000000254"
-"000000020000000000000255"
-"000000020000000000000256"
-"000000020000000000000257"
-"000000020000000000000258"
-"000000020000000000000259"
-"00000002000000000000025A"
-"00000002000000000000025B"
-"00000002000000000000025C"
-"00000002000000000000025D"
-"00000002000000000000025E"
-"00000002000000000000025F"
-"000000020000000000000260"
-"000000020000000000000261"
-"000000020000000000000262"
-"000000020000000000000263"
-"000000020000000000000264"
-"000000020000000000000265"
-"000000020000000000000266"
-"000000020000000000000267"
-"000000020000000000000268"
-"000000020000000000000269"
-"00000002000000000000026A"
-"00000002000000000000026B"
-"00000002000000000000026C"
-"00000002000000000000026D"
-"00000002000000000000026E"
-"00000002000000000000026F"
-"000000020000000000000270"
-"000000020000000000000271"
-"000000020000000000000272"
-"000000020000000000000273"
-"000000020000000000000274"
-"000000020000000000000275"
-"000000020000000000000276"
-"000000020000000000000277"
-"000000020000000000000278"
-"000000020000000000000279"
-"00000002000000000000027A"
-"00000002000000000000027B"
-"00000002000000000000027C"
-"00000002000000000000027D"
-"00000002000000000000027E"
-"00000002000000000000027F"
-"000000020000000000000280"
-"000000020000000000000281"
-"000000020000000000000282"
-"000000020000000000000283"
-"000000020000000000000284"
-"000000020000000000000285"
-"000000020000000000000286"
-"000000020000000000000287"
-"000000020000000000000288"
-"000000020000000000000289"
-"00000002000000000000028A"
-"00000002000000000000028B"
-"00000002000000000000028C"
-"00000002000000000000028D"
-"00000002000000000000028E"
-"00000002000000000000028F"
-"000000020000000000000290"
-"000000020000000000000291"
-"000000020000000000000292"
-"000000020000000000000293"
-"000000020000000000000294"
-"000000020000000000000295"
-"000000020000000000000296"
-"000000020000000000000297"
-"000000020000000000000298"
-"000000020000000000000299"
-"00000002000000000000029A"
-"00000002000000000000029B"
-"00000002000000000000029C"
-"00000002000000000000029D"
-"00000002000000000000029E"
-"00000002000000000000029F"
-"0000000200000000000002A0"
-"0000000200000000000002A1"
-"0000000200000000000002A2"
-"0000000200000000000002A3"
-"0000000200000000000002A4"
-"0000000200000000000002A5"
-"0000000200000000000002A6"
-"0000000200000000000002A7"
-"0000000200000000000002A8"
-"0000000200000000000002A9"
-"0000000200000000000002AA"
-"0000000200000000000002AB"
-"0000000200000000000002AC"
-"0000000200000000000002AD"
-"0000000200000000000002AE"
-"0000000200000000000002AF"
-"0000000200000000000002B0"
-"0000000200000000000002B1"
-"0000000200000000000002B2"
-"0000000200000000000002B3"
-"0000000200000000000002B4"
-"0000000200000000000002B5"
-"0000000200000000000002B6"
-"0000000200000000000002B7"
-"0000000200000000000002B8"
-"0000000200000000000002B9"
-"0000000200000000000002BA"
-"0000000200000000000002BB"
-"0000000200000000000002BC"
-"0000000200000000000002BD"
-"0000000200000000000002BE"
-"0000000200000000000002BF"
-"0000000200000000000002C0"
-"0000000200000000000002C1"
-"0000000200000000000002C2"
-"0000000200000000000002C3"
-"0000000200000000000002C4"
-"0000000200000000000002C5"
-"0000000200000000000002C6"
-"0000000200000000000002C7"
-"0000000200000000000002C8"
-"0000000200000000000002C9"
-"0000000200000000000002CA"
-"0000000200000000000002CB"
-"0000000200000000000002CC"
-"0000000200000000000002CD"
-"0000000200000000000002CE"
-"0000000200000000000002CF"
-"0000000200000000000002D0"
-"0000000200000000000002D1"
-"0000000200000000000002D2"
-"0000000200000000000002D3"
-"0000000200000000000002D4"
-"0000000200000000000002D5"
-"0000000200000000000002D6"
-"0000000200000000000002D7"
-"0000000200000000000002D8"
-"0000000200000000000002D9"
-"0000000200000000000002DA"
-"0000000200000000000002DB"
-"0000000200000000000002DC"
-"0000000200000000000002DD"
-"0000000200000000000002DE"
-"0000000200000000000002DF"
-"0000000200000000000002E0"
-"0000000200000000000002E1"
-"0000000200000000000002E2"
-"0000000200000000000002E3"
-"0000000200000000000002E4"
-"0000000200000000000002E5"
-"0000000200000000000002E6"
-"0000000200000000000002E7"
-"0000000200000000000002E8"
-"0000000200000000000002E9"
-"0000000200000000000002EA"
-"0000000200000000000002EB"
-"0000000200000000000002EC"
-"0000000200000000000002ED"
-"0000000200000000000002EE"
-"0000000200000000000002EF"
-"0000000200000000000002F0"
-"0000000200000000000002F1"
-"0000000200000000000002F2"
-"0000000200000000000002F3"
-"0000000200000000000002F4"
-"0000000200000000000002F5"
-"0000000200000000000002F6"
-"0000000200000000000002F7"
-"0000000200000000000002F8"
-"0000000200000000000002F9"
-"0000000200000000000002FA"
-"0000000200000000000002FB"
-"0000000200000000000002FC"
-"0000000200000000000002FD"
-"0000000200000000000002FE"
-"0000000200000000000002FF"
-"000000020000000000000300"
-"000000020000000000000301"
-"000000020000000000000302"
-"000000020000000000000303"
-"000000020000000000000304"
-"000000020000000000000305"
-"000000020000000000000306"
-"000000020000000000000307"
-"000000020000000000000308"
-"000000020000000000000309"
-"00000002000000000000030A"
-"00000002000000000000030B"
-"00000002000000000000030C"
-"00000002000000000000030D"
-"00000002000000000000030E"
-"00000002000000000000030F"
-"000000020000000000000310"
-"000000020000000000000311"
-"000000020000000000000312"
-"000000020000000000000313"
-"000000020000000000000314"
-"000000020000000000000315"
-"000000020000000000000316"
-"000000020000000000000317"
-"000000020000000000000318"
-"000000020000000000000319"
-"00000002000000000000031A"
-"00000002000000000000031B"
-"00000002000000000000031C"
-"00000002000000000000031D"
-"00000002000000000000031E"
-"00000002000000000000031F"
-"000000020000000000000320"
-"000000020000000000000321"
-"000000020000000000000322"
-"000000020000000000000323"
-"000000020000000000000324"
-"000000020000000000000325"
-"000000020000000000000326"
-"000000020000000000000327"
-"000000020000000000000328"
-"000000020000000000000329"
-"00000002000000000000032A"
-"00000002000000000000032B"
-"00000002000000000000032C"
-"00000002000000000000032D"
-"00000002000000000000032E"
-"00000002000000000000032F"
-"000000020000000000000330"
-"000000020000000000000331"
-"000000020000000000000332"
-"000000020000000000000333"
-"000000020000000000000334"
-"000000020000000000000335"
-"000000020000000000000336"
-"000000020000000000000337"
-"000000020000000000000338"
-"000000020000000000000339"
-"00000002000000000000033A"
-"00000002000000000000033B"
-"00000002000000000000033C"
-"00000002000000000000033D"
-"00000002000000000000033E"
-"00000002000000000000033F"
-"000000020000000000000340"
-"000000020000000000000341"
-"000000020000000000000342"
-"000000020000000000000343"
-"000000020000000000000344"
-"000000020000000000000345"
-"000000020000000000000346"
-"000000020000000000000347"
-"000000020000000000000348"
-"000000020000000000000349"
-"00000002000000000000034A"
-"00000002000000000000034B"
-"00000002000000000000034C"
-"00000002000000000000034D"
-"00000002000000000000034E"
-"00000002000000000000034F"
-"000000020000000000000350"
-"000000020000000000000351"
-"000000020000000000000352"
-"000000020000000000000353"
-"000000020000000000000354"
-"000000020000000000000355"
-"000000020000000000000356"
-"000000020000000000000357"
-"000000020000000000000358"
-"000000020000000000000359"
-"00000002000000000000035A"
-"00000002000000000000035B"
-"00000002000000000000035C"
-"00000002000000000000035D"
-"00000002000000000000035E"
-"00000002000000000000035F"
-"000000020000000000000360"
-"000000020000000000000361"
-"000000020000000000000362"
-"000000020000000000000363"
-"000000020000000000000364"
-"000000020000000000000365"
-"000000020000000000000366"
-"000000020000000000000367"
-"000000020000000000000368"
-"000000020000000000000369"
-"00000002000000000000036A"
-"00000002000000000000036B"
-"00000002000000000000036C"
-"00000002000000000000036D"
-"00000002000000000000036E"
-"00000002000000000000036F"
-"000000020000000000000370"
-"000000020000000000000371"
-"000000020000000000000372"
-"000000020000000000000373"
-"000000020000000000000374"
-"000000020000000000000375"
-"000000020000000000000376"
-"000000020000000000000377"
-"000000020000000000000378"
-"000000020000000000000379"
-"00000002000000000000037A"
-"00000002000000000000037B"
-"00000002000000000000037C"
-"00000002000000000000037D"
-"00000002000000000000037E"
-"00000002000000000000037F"
-"000000020000000000000380"
-"000000020000000000000381"
-"000000020000000000000382"
-"000000020000000000000383"
-"000000020000000000000384"
-"000000020000000000000385"
-"000000020000000000000386"
-"000000020000000000000387"
-"000000020000000000000388"
-"000000020000000000000389"
-"00000002000000000000038A"
-"00000002000000000000038B"
-"00000002000000000000038C"
-"00000002000000000000038D"
-"00000002000000000000038E"
-"00000002000000000000038F"
-"000000020000000000000390"
-"000000020000000000000391"
-"000000020000000000000392"
-"000000020000000000000393"
-"000000020000000000000394"
-"000000020000000000000395"
-"000000020000000000000396"
-"000000020000000000000397"
-"000000020000000000000398"
-"000000020000000000000399"
-"00000002000000000000039A"
-"00000002000000000000039B"
-"00000002000000000000039C"
-"00000002000000000000039D"
-"00000002000000000000039E"
-"00000002000000000000039F"
-"0000000200000000000003A0"
-"0000000200000000000003A1"
-"0000000200000000000003A2"
-"0000000200000000000003A3"
-"0000000200000000000003A4"
-"0000000200000000000003A5"
-"0000000200000000000003A6"
-"0000000200000000000003A7"
-"0000000200000000000003A8"
-"0000000200000000000003A9"
-"0000000200000000000003AA"
-"0000000200000000000003AB"
-"0000000200000000000003AC"
-"0000000200000000000003AD"
-"0000000200000000000003AE"
-"0000000200000000000003AF"
-"0000000200000000000003B0"
-"0000000200000000000003B1"
-"0000000200000000000003B2"
-"0000000200000000000003B3"
-"0000000200000000000003B4"
-"0000000200000000000003B5"
-"0000000200000000000003B6"
-"0000000200000000000003B7"
-"0000000200000000000003B8"
-"0000000200000000000003B9"
-"0000000200000000000003BA"
-"0000000200000000000003BB"
-"0000000200000000000003BC"
-"0000000200000000000003BD"
-"0000000200000000000003BE"
-"0000000200000000000003BF"
-"0000000200000000000003C0"
-"0000000200000000000003C1"
-"0000000200000000000003C2"
-"0000000200000000000003C3"
-"0000000200000000000003C4"
-"0000000200000000000003C5"
-"0000000200000000000003C6"
-"0000000200000000000003C7"
-"0000000200000000000003C8"
-"0000000200000000000003C9"
-"0000000200000000000003CA"
-"0000000200000000000003CB"
-"0000000200000000000003CC"
-"0000000200000000000003CD"
-"0000000200000000000003CE"
-"0000000200000000000003CF"
-"0000000200000000000003D0"
-"0000000200000000000003D1"
-"0000000200000000000003D2"
-"0000000200000000000003D3"
-"0000000200000000000003D4"
-"0000000200000000000003D5"
-"0000000200000000000003D6"
-"0000000200000000000003D7"
-"0000000200000000000003D8"
-"0000000200000000000003D9"
-"0000000200000000000003DA"
-"0000000200000000000003DB"
-"0000000200000000000003DC"
-"0000000200000000000003DD"
-"0000000200000000000003DE"
-"0000000200000000000003DF"
-"0000000200000000000003E0"
-"0000000200000000000003E1"
-"0000000200000000000003E2"
-"0000000200000000000003E3"
-"0000000200000000000003E4"
-"0000000200000000000003E5"
-"0000000200000000000003E6"
-"0000000200000000000003E7"
-"0000000200000000000003E8"
-"0000000200000000000003E9"
-"0000000200000000000003EA"
-"0000000200000000000003EB"
-"0000000200000000000003EC"
-"0000000200000000000003ED"
-"0000000200000000000003EE"
-"0000000200000000000003EF"
-"0000000200000000000003F0"
-"0000000200000000000003F1"
-"0000000200000000000003F2"
-"0000000200000000000003F3"
-"0000000200000000000003F4"
-"0000000200000000000003F5"
-"0000000200000000000003F6"
-"0000000200000000000003F7"
-"0000000200000000000003F8"
-"0000000200000000000003F9"
-"0000000200000000000003FA"
-"0000000200000000000003FB"
-"0000000200000000000003FC"
-"0000000200000000000003FD"
-"0000000200000000000003FE"
-"0000000200000000000003FF"
-"000000020000000000000400"
-"000000020000000000000401"
-"000000020000000000000402"
-"000000020000000000000403"
-"000000020000000000000404"
-"000000020000000000000405"
-"000000020000000000000406"
-"000000020000000000000407"
-"000000020000000000000408"
-"000000020000000000000409"
-"00000002000000000000040A"
-"00000002000000000000040B"
-"00000002000000000000040C"
-"00000002000000000000040D"
-"00000002000000000000040E"
-"00000002000000000000040F"
-"000000020000000000000410"
-"000000020000000000000411"
-"000000020000000000000412"
-"000000020000000000000413"
-"000000020000000000000414"
-"000000020000000000000415"
-"000000020000000000000416"
-"000000020000000000000417"
-"000000020000000000000418"
-"000000020000000000000419"
-"00000002000000000000041A"
-"00000002000000000000041B"
-"00000002000000000000041C"
-"00000002000000000000041D"
-"00000002000000000000041E"
-"00000002000000000000041F"
-"000000020000000000000420"
-"000000020000000000000421"
-"000000020000000000000422"
-"000000020000000000000423"
-"000000020000000000000424"
-"000000020000000000000425"
-"000000020000000000000426"
-"000000020000000000000427"
-"000000020000000000000428"
-"000000020000000000000429"
-"00000002000000000000042A"
-"00000002000000000000042B"
-"00000002000000000000042C"
-"00000002000000000000042D"
-"00000002000000000000042E"
-"00000002000000000000042F"
-"000000020000000000000430"
-"000000020000000000000431"
-"000000020000000000000432"
-"000000020000000000000433"
-"000000020000000000000434"
-"000000020000000000000435"
-"000000020000000000000436"
-"000000020000000000000437"
-"000000020000000000000438"
-"000000020000000000000439"
-"00000002000000000000043A"
-"00000002000000000000043B"
-"00000002000000000000043C"
-"00000002000000000000043D"
-"00000002000000000000043E"
-"00000002000000000000043F"
-"000000020000000000000440"
-"000000020000000000000441"
-"000000020000000000000442"
-"000000020000000000000443"
-"000000020000000000000444"
-"000000020000000000000445"
-"000000020000000000000446"
-"000000020000000000000447"
-"000000020000000000000448"
-"000000020000000000000449"
-"00000002000000000000044A"
-"00000002000000000000044B"
-"00000002000000000000044C"
-"00000002000000000000044D"
-"00000002000000000000044E"
-"00000002000000000000044F"
-"000000020000000000000450"
-"000000020000000000000451"
-"000000020000000000000452"
-"000000020000000000000453"
-"000000020000000000000454"
-"000000020000000000000455"
-"000000020000000000000456"
-"000000020000000000000457"
-"000000020000000000000458"
-"000000020000000000000459"
-"00000002000000000000045A"
-"00000002000000000000045B"
-"00000002000000000000045C"
-"00000002000000000000045D"
-"00000002000000000000045E"
-"00000002000000000000045F"
-"000000020000000000000460"
-"000000020000000000000461"
-"000000020000000000000462"
-"000000020000000000000463"
-"000000020000000000000464"
-"000000020000000000000465"
-"000000020000000000000466"
-"000000020000000000000467"
-"000000020000000000000468"
-"000000020000000000000469"
-"00000002000000000000046A"
-"00000002000000000000046B"
-"00000002000000000000046C"
-"00000002000000000000046D"
-"00000002000000000000046E"
-"00000002000000000000046F"
-"000000020000000000000470"
-"000000020000000000000471"
-"000000020000000000000472"
-"000000020000000000000473"
-"000000020000000000000474"
-"000000020000000000000475"
-"000000020000000000000476"
-"000000020000000000000477"
-"000000020000000000000478"
-"000000020000000000000479"
-"00000002000000000000047A"
-"00000002000000000000047B"
-"00000002000000000000047C"
-"00000002000000000000047D"
-"00000002000000000000047E"
-"00000002000000000000047F"
-"000000020000000000000480"
-"000000020000000000000481"
-"000000020000000000000482"
-"000000020000000000000483"
-"000000020000000000000484"
-"000000020000000000000485"
-"000000020000000000000486"
-"000000020000000000000487"
-"000000020000000000000488"
-"000000020000000000000489"
-"00000002000000000000048A"
-"00000002000000000000048B"
-"00000002000000000000048C"
-"00000002000000000000048D"
-"00000002000000000000048E"
-"00000002000000000000048F"
-"000000020000000000000490"
-"000000020000000000000491"
-"000000020000000000000492"
-"000000020000000000000493"
-"000000020000000000000494"
-"000000020000000000000495"
-"000000020000000000000496"
-"000000020000000000000497"
-"000000020000000000000498"
-"000000020000000000000499"
-"00000002000000000000049A"
-"00000002000000000000049B"
-"00000002000000000000049C"
-"00000002000000000000049D"
-"00000002000000000000049E"
-"00000002000000000000049F"
-"0000000200000000000004A0"
-"0000000200000000000004A1"
-"0000000200000000000004A2"
-"0000000200000000000004A3"
-"0000000200000000000004A4"
-"0000000200000000000004A5"
-"0000000200000000000004A6"
-"0000000200000000000004A7"
-"0000000200000000000004A8"
-"0000000200000000000004A9"
-"0000000200000000000004AA"
-"0000000200000000000004AB"
-"0000000200000000000004AC"
-"0000000200000000000004AD"
-"0000000200000000000004AE"
-"0000000200000000000004AF"
-"0000000200000000000004B0"
-"0000000200000000000004B1"
-"0000000200000000000004B2"
-"0000000200000000000004B3"
-"0000000200000000000004B4"
-"0000000200000000000004B5"
-"0000000200000000000004B6"
-"0000000200000000000004B7"
-"0000000200000000000004B8"
-"0000000200000000000004B9"
-"0000000200000000000004BA"
-"0000000200000000000004BB"
-"0000000200000000000004BC"
-"0000000200000000000004BD"
-"0000000200000000000004BE"
-"0000000200000000000004BF"
-"0000000200000000000004C0"
-"0000000200000000000004C1"
-"0000000200000000000004C2"
-"0000000200000000000004C3"
-"0000000200000000000004C4"
-"0000000200000000000004C5"
-"0000000200000000000004C6"
-"0000000200000000000004C7"
-"0000000200000000000004C8"
-"0000000200000000000004C9"
-"0000000200000000000004CA"
-"0000000200000000000004CB"
-"0000000200000000000004CC"
-"0000000200000000000004CD"
-"0000000200000000000004CE"
-"0000000200000000000004CF"
-"0000000200000000000004D0"
-"0000000200000000000004D1"
-"0000000200000000000004D2"
-"0000000200000000000004D3"
-"0000000200000000000004D4"
-"0000000200000000000004D5"
-"0000000200000000000004D6"
-"0000000200000000000004D7"
-"0000000200000000000004D8"
-"0000000200000000000004D9"
-"0000000200000000000004DA"
-"0000000200000000000004DB"
-"0000000200000000000004DC"
-"0000000200000000000004DD"
-"0000000200000000000004DE"
-"0000000200000000000004DF"
-"0000000200000000000004E0"
-"0000000200000000000004E1"
-"0000000200000000000004E2"
-"0000000200000000000004E3"
-"0000000200000000000004E4"
-"0000000200000000000004E5"
-"0000000200000000000004E6"
-"0000000200000000000004E7"
-"0000000200000000000004E8"
-"0000000200000000000004E9"
-"0000000200000000000004EA"
-"0000000200000000000004EB"
-"0000000200000000000004EC"
-"0000000200000000000004ED"
-"0000000200000000000004EE"
-"0000000200000000000004EF"
-"0000000200000000000004F0"
-"0000000200000000000004F1"
-"0000000200000000000004F2"
-"0000000200000000000004F3"
-"0000000200000000000004F4"
-"0000000200000000000004F5"
-"0000000200000000000004F6"
-"0000000200000000000004F7"
-"0000000200000000000004F8"
-"0000000200000000000004F9"
-"0000000200000000000004FA"
-"0000000200000000000004FB"
-"0000000200000000000004FC"
-"0000000200000000000004FD"
-"0000000200000000000004FE"
-"0000000200000000000004FF"
-"000000020000000000000500"
-"000000020000000000000501"
-"000000020000000000000502"
-"000000020000000000000503"
-"000000020000000000000504"
-"000000020000000000000505"
-"000000020000000000000506"
-"000000020000000000000507"
-"000000020000000000000508"
-"000000020000000000000509"
-"00000002000000000000050A"
-"00000002000000000000050B"
-"00000002000000000000050C"
-"00000002000000000000050D"
-"00000002000000000000050E"
-"00000002000000000000050F"
-"000000020000000000000510"
-"000000020000000000000511"
-"000000020000000000000512"
-"000000020000000000000513"
-"000000020000000000000514"
-"000000020000000000000515"
-"000000020000000000000516"
-"000000020000000000000517"
-"000000020000000000000518"
-"000000020000000000000519"
-"00000002000000000000051A"
-"00000002000000000000051B"
-"00000002000000000000051C"
-"00000002000000000000051D"
-"00000002000000000000051E"
-"00000002000000000000051F"
-"000000020000000000000520"
-"000000020000000000000521"
-"000000020000000000000522"
-"000000020000000000000523"
-"000000020000000000000524"
-"000000020000000000000525"
-"000000020000000000000526"
-"000000020000000000000527"
-"000000020000000000000528"
-"000000020000000000000529"
-"00000002000000000000052A"
-"00000002000000000000052B"
-"00000002000000000000052C"
-"00000002000000000000052D"
-"00000002000000000000052E"
-"00000002000000000000052F"
-"000000020000000000000530"
-"000000020000000000000531"
-"000000020000000000000532"
-"000000020000000000000533"
-"000000020000000000000534"
-"000000020000000000000535"
-"000000020000000000000536"
-"000000020000000000000537"
-"000000020000000000000538"
-"000000020000000000000539"
-"00000002000000000000053A"
-"00000002000000000000053B"
-"00000002000000000000053C"
-"00000002000000000000053D"
-"00000002000000000000053E"
-"00000002000000000000053F"
-"000000020000000000000540"
-"000000020000000000000541"
-"000000020000000000000542"
-"000000020000000000000543"
-"000000020000000000000544"
-"000000020000000000000545"
-"000000020000000000000546"
-"000000020000000000000547"
-"000000020000000000000548"
-"000000020000000000000549"
-"00000002000000000000054A"
-"00000002000000000000054B"
-"00000002000000000000054C"
-"00000002000000000000054D"
-"00000002000000000000054E"
-"00000002000000000000054F"
-"000000020000000000000550"
-"000000020000000000000551"
-"000000020000000000000552"
-"000000020000000000000553"
-"000000020000000000000554"
-"000000020000000000000555"
-"000000020000000000000556"
-"000000020000000000000557"
-"000000020000000000000558"
-"000000020000000000000559"
-"00000002000000000000055A"
-"00000002000000000000055B"
-"00000002000000000000055C"
-"00000002000000000000055D"
-"00000002000000000000055E"
-"00000002000000000000055F"
-"000000020000000000000560"
-"000000020000000000000561"
-"000000020000000000000562"
-"000000020000000000000563"
-"000000020000000000000564"
-"000000020000000000000565"
-"000000020000000000000566"
-"000000020000000000000567"
-"000000020000000000000568"
-"000000020000000000000569"
-"00000002000000000000056A"
-"00000002000000000000056B"
-"00000002000000000000056C"
-"00000002000000000000056D"
-"00000002000000000000056E"
-"00000002000000000000056F"
-"000000020000000000000570"
-"000000020000000000000571"
-"000000020000000000000572"
-"000000020000000000000573"
-"000000020000000000000574"
-"000000020000000000000575"
-"000000020000000000000576"
-"000000020000000000000577"
-"000000020000000000000578"
-"000000020000000000000579"
-"00000002000000000000057A"
-"00000002000000000000057B"
-"00000002000000000000057C"
-"00000002000000000000057D"
-"00000002000000000000057E"
-"00000002000000000000057F"
-"000000020000000000000580"
-"000000020000000000000581"
-"000000020000000000000582"
-"000000020000000000000583"
-"000000020000000000000584"
-"000000020000000000000585"
-"000000020000000000000586"
-"000000020000000000000587"
-"000000020000000000000588"
-"000000020000000000000589"
-"00000002000000000000058A"
-"00000002000000000000058B"
-"00000002000000000000058C"
-"00000002000000000000058D"
-"00000002000000000000058E"
-"00000002000000000000058F"
-"000000020000000000000590"
-"000000020000000000000591"
-"000000020000000000000592"
-"000000020000000000000593"
-"000000020000000000000594"
-"000000020000000000000595"
-"000000020000000000000596"
-"000000020000000000000597"
-"000000020000000000000598"
-"000000020000000000000599"
-"00000002000000000000059A"
-"00000002000000000000059B"
-"00000002000000000000059C"
-"00000002000000000000059D"
-"00000002000000000000059E"
-"00000002000000000000059F"
-"0000000200000000000005A0"
-"0000000200000000000005A1"
-"0000000200000000000005A2"
-"0000000200000000000005A3"
-"0000000200000000000005A4"
-"0000000200000000000005A5"
-"0000000200000000000005A6"
-"0000000200000000000005A7"
-"0000000200000000000005A8"
-"0000000200000000000005A9"
-"0000000200000000000005AA"
-"0000000200000000000005AB"
-"0000000200000000000005AC"
-"0000000200000000000005AD"
-"0000000200000000000005AE"
-"0000000200000000000005AF"
-"0000000200000000000005B0"
-"0000000200000000000005B1"
-"0000000200000000000005B2"
-"0000000200000000000005B3"
-"0000000200000000000005B4"
-"0000000200000000000005B5"
-"0000000200000000000005B6"
-"0000000200000000000005B7"
-"0000000200000000000005B8"
-"0000000200000000000005B9"
-"0000000200000000000005BA"
-"000000030000000000000001"
-"000000030000000000000002"
-"000000030000000000000003"
-"000000030000000000000004"
-"000000030000000000000005"
-"000000030000000000000006"
-"000000030000000000000007"
-"000000030000000000000008"
-"000000030000000000000009"
-"00000003000000000000000A"
-"00000003000000000000000B"
-"00000003000000000000000C"
-"00000003000000000000000D"
-"00000003000000000000000E"
-"00000003000000000000000F"
-"000000030000000000000010"
-"000000030000000000000011"
-"000000030000000000000012"
-"000000030000000000000013"
-"000000030000000000000014"
-"000000030000000000000015"
-"000000030000000000000016"
-"000000030000000000000017"
-"000000030000000000000018"
-"000000030000000000000019"
-"00000003000000000000001A"
-"00000003000000000000001B"
-"00000003000000000000001C"
-"00000003000000000000001D"
-"00000003000000000000001E"
-"00000003000000000000001F"
-"000000030000000000000020"
-"000000030000000000000021"
-"000000030000000000000022"
-"000000030000000000000023"
-"000000030000000000000024"
-"000000030000000000000025"
-"000000030000000000000026"
-"000000030000000000000027"
-"000000030000000000000028"
-"000000030000000000000029"
-"00000003000000000000002A"
-"00000003000000000000002B"
-"00000003000000000000002C"
-"00000003000000000000002D"
-"00000003000000000000002E"
-"00000003000000000000002F"
-"000000030000000000000030"
-"000000030000000000000031"
-"000000030000000000000032"
-"000000030000000000000033"
-"000000030000000000000034"
-"000000030000000000000035"
-"000000030000000000000036"
-"000000030000000000000037"
-"000000030000000000000038"
-"000000030000000000000039"
-"00000003000000000000003A"
-"00000003000000000000003B"
-"00000003000000000000003C"
-"00000003000000000000003D"
-"00000003000000000000003E"
-"00000003000000000000003F"
-"000000030000000000000040"
-"000000030000000000000041"
-"000000030000000000000042"
-"000000030000000000000043"
-"000000030000000000000044"
-"000000030000000000000045"
-"000000030000000000000046"
-"000000030000000000000047"
-"000000030000000000000048"
-"000000030000000000000049"
-"00000003000000000000004A"
-"00000003000000000000004B"
-"00000003000000000000004C"
-"00000003000000000000004D"
-"00000003000000000000004E"
-"00000003000000000000004F"
-"000000030000000000000050"
-"000000030000000000000051"
-"000000030000000000000052"
-"000000030000000000000053"
-"000000030000000000000054"
-"000000030000000000000055"
-"000000030000000000000056"
-"000000030000000000000057"
-"000000030000000000000058"
-"000000030000000000000059"
-"00000003000000000000005A"
-"00000003000000000000005B"
-"00000003000000000000005C"
-"00000003000000000000005D"
-"00000003000000000000005E"
-"00000003000000000000005F"
-"000000030000000000000060"
-"000000030000000000000061"
-"000000030000000000000062"
-"000000030000000000000063"
-"000000030000000000000064"
-"000000030000000000000065"
-"000000030000000000000066"
-"000000030000000000000067"
-"000000030000000000000068"
-"000000030000000000000069"
-"00000003000000000000006A"
-"00000003000000000000006B"
-"00000003000000000000006C"
-"00000003000000000000006D"
-"00000003000000000000006E"
-"00000003000000000000006F"
-"000000030000000000000070"
-"000000030000000000000071"
-"000000030000000000000072"
-"000000030000000000000073"
-"000000030000000000000074"
-"000000030000000000000075"
-"000000030000000000000076"
-"000000030000000000000077"
-"000000030000000000000078"
-"000000030000000000000079"
-"00000003000000000000007A"
-"00000003000000000000007B"
-"00000003000000000000007C"
-"00000003000000000000007D"
-"00000003000000000000007E"
-"00000003000000000000007F"
-"000000030000000000000080"
-"000000030000000000000081"
-"000000030000000000000082"
-"000000030000000000000083"
-"000000030000000000000084"
-"000000030000000000000085"
-"000000030000000000000086"
-"000000030000000000000087"
-"000000030000000000000088"
-"000000030000000000000089"
-"00000003000000000000008A"
-"00000003000000000000008B"
-"00000003000000000000008C"
-"00000003000000000000008D"
-"00000003000000000000008E"
-"00000003000000000000008F"
-"000000030000000000000090"
-"000000030000000000000091"
-"000000030000000000000092"
-"000000030000000000000093"
-"000000030000000000000094"
-"000000030000000000000095"
-"000000030000000000000096"
-"000000030000000000000097"
-"000000030000000000000098"
-"000000030000000000000099"
-"00000003000000000000009A"
-"00000003000000000000009B"
-"00000003000000000000009C"
-"00000003000000000000009D"
-"00000003000000000000009E"
-"00000003000000000000009F"
-"0000000300000000000000A0"
-"0000000300000000000000A1"
-"0000000300000000000000A2"
-"0000000300000000000000A3"
-"0000000300000000000000A4"
-"0000000300000000000000A5"
-"0000000300000000000000A6"
-"0000000300000000000000A7"
-"0000000300000000000000A8"
-"0000000300000000000000A9"
-"0000000300000000000000AA"
-"0000000300000000000000AB"
-"0000000300000000000000AC"
-"0000000300000000000000AD"
-"0000000300000000000000AE"
-"0000000300000000000000AF"
-"0000000300000000000000B0"
-"0000000300000000000000B1"
-"0000000300000000000000B2"
-"0000000300000000000000B3"
-"0000000300000000000000B4"
-"0000000300000000000000B5"
-"0000000300000000000000B6"
-"0000000300000000000000B7"
-"0000000300000000000000B8"
-"0000000300000000000000B9"
-"0000000300000000000000BA"
-"0000000300000000000000BB"
-"0000000300000000000000BC"
-"0000000300000000000000BD"
-"0000000300000000000000BE"
-"0000000300000000000000BF"
-"0000000300000000000000C0"
-"0000000300000000000000C1"
-"0000000300000000000000C2"
-"0000000300000000000000C3"
-"0000000300000000000000C4"
-"0000000300000000000000C5"
-"0000000300000000000000C6"
-"0000000300000000000000C7"
-"0000000300000000000000C8"
-"0000000300000000000000C9"
-"0000000300000000000000CA"
-"0000000300000000000000CB"
-"0000000300000000000000CC"
-"0000000300000000000000CD"
-"0000000300000000000000CE"
-"0000000300000000000000CF"
-"0000000300000000000000D0"
-"0000000300000000000000D1"
-"0000000300000000000000D2"
-"0000000300000000000000D3"
-"0000000300000000000000D4"
-"0000000300000000000000D5"
-"0000000300000000000000D6"
-"0000000300000000000000D7"
-"0000000300000000000000D8"
-"0000000300000000000000D9"
-"0000000300000000000000DA"
-"0000000300000000000000DB"
-"0000000300000000000000DC"
-"0000000300000000000000DD"
-"0000000300000000000000DE"
-"0000000300000000000000DF"
-"0000000300000000000000E0"
-"0000000300000000000000E1"
-"0000000300000000000000E2"
-"0000000300000000000000E3"
-"0000000300000000000000E4"
-"0000000300000000000000E5"
-"0000000300000000000000E6"
-"0000000300000000000000E7"
-"0000000300000000000000E8"
-"0000000300000000000000E9"
-"0000000300000000000000EA"
-"0000000300000000000000EB"
-"0000000300000000000000EC"
-"0000000300000000000000ED"
-"0000000300000000000000EE"
-"0000000300000000000000EF"
-"0000000300000000000000F0"
-"0000000300000000000000F1"
-"0000000300000000000000F2"
-"0000000300000000000000F3"
-"0000000300000000000000F4"
-"0000000300000000000000F5"
-"0000000300000000000000F6"
-"0000000300000000000000F7"
-"0000000300000000000000F8"
-"0000000300000000000000F9"
-"0000000300000000000000FA"
-"0000000300000000000000FB"
-"0000000300000000000000FC"
-"0000000300000000000000FD"
-"0000000300000000000000FE"
-"0000000300000000000000FF"
-"000000030000000000000100"
-"000000030000000000000101"
-"000000030000000000000102"
-"000000030000000000000103"
-"000000030000000000000104"
-"000000030000000000000105"
-"000000030000000000000106"
-"000000030000000000000107"
-"000000030000000000000108"
-"000000030000000000000109"
-"00000003000000000000010A"
-"00000003000000000000010B"
-"00000003000000000000010C"
-"00000003000000000000010D"
-"00000003000000000000010E"
-"00000003000000000000010F"
-"000000030000000000000110"
-"000000030000000000000111"
-"000000030000000000000112"
-"000000030000000000000113"
-"000000030000000000000114"
-"000000030000000000000115"
-"000000030000000000000116"
-"000000030000000000000117"
-"000000030000000000000118"
-"000000030000000000000119"
-"00000003000000000000011A"
-"00000003000000000000011B"
-"00000003000000000000011C"
-"00000003000000000000011D"
-"00000003000000000000011E"
-"00000003000000000000011F"
-"000000030000000000000120"
-"000000030000000000000121"
-"000000030000000000000122"
-"000000030000000000000123"
-"000000030000000000000124"
-"000000030000000000000125"
-"000000030000000000000126"
-"000000030000000000000127"
-"000000030000000000000128"
-"000000030000000000000129"
-"00000003000000000000012A"
-"00000003000000000000012B"
-"00000003000000000000012C"
-"00000003000000000000012D"
-"00000003000000000000012E"
-"00000003000000000000012F"
-"000000030000000000000130"
-"000000030000000000000131"
-"000000030000000000000132"
-"000000030000000000000133"
-"000000030000000000000134"
-"000000030000000000000135"
-"000000030000000000000136"
-"000000030000000000000137"
-"000000030000000000000138"
-"000000030000000000000139"
-"00000003000000000000013A"
-"00000003000000000000013B"
-"00000003000000000000013C"
-"00000003000000000000013D"
-"00000003000000000000013E"
-"00000003000000000000013F"
-"000000030000000000000140"
-"000000030000000000000141"
-"000000030000000000000142"
-"000000030000000000000143"
-"000000030000000000000144"
-"000000030000000000000145"
-"000000030000000000000146"
-"000000030000000000000147"
-"000000030000000000000148"
-"000000030000000000000149"
-"00000003000000000000014A"
-"00000003000000000000014B"
-"00000003000000000000014C"
-"00000003000000000000014D"
-"00000003000000000000014E"
-"00000003000000000000014F"
-"000000030000000000000150"
-"000000030000000000000151"
-"000000030000000000000152"
-"000000030000000000000153"
-"000000030000000000000154"
-"000000030000000000000155"
-"000000030000000000000156"
-"000000030000000000000157"
-"000000030000000000000158"
-"000000030000000000000159"
-"00000003000000000000015A"
-"00000003000000000000015B"
-"00000003000000000000015C"
-"00000003000000000000015D"
-"00000003000000000000015E"
-"00000003000000000000015F"
-"000000030000000000000160"
-"000000030000000000000161"
-"000000030000000000000162"
-"000000030000000000000163"
-"000000030000000000000164"
-"000000030000000000000165"
-"000000030000000000000166"
-"000000030000000000000167"
-"000000030000000000000168"
-"000000030000000000000169"
-"00000003000000000000016A"
-"00000003000000000000016B"
-"00000003000000000000016C"
-"00000003000000000000016D"
-"00000003000000000000016E"
-"00000003000000000000016F"
-"000000030000000000000170"
-"000000030000000000000171"
-"000000030000000000000172"
-"000000030000000000000173"
-"000000030000000000000174"
-"000000030000000000000175"
-"000000030000000000000176"
-"000000030000000000000177"
-"000000030000000000000178"
-"000000030000000000000179"
-"00000003000000000000017A"
-"00000003000000000000017B"
-"00000003000000000000017C"
-"00000003000000000000017D"
-"00000003000000000000017E"
-"00000003000000000000017F"
-"000000030000000000000180"
-"000000030000000000000181"
-"000000030000000000000182"
-"000000030000000000000183"
-"000000030000000000000184"
-"000000030000000000000185"
-"000000030000000000000186"
-"000000030000000000000187"
-"000000030000000000000188"
-"000000030000000000000189"
-"00000003000000000000018A"
-"00000003000000000000018B"
-"00000003000000000000018C"
-"00000003000000000000018D"
-"00000003000000000000018E"
-"00000003000000000000018F"
-"000000030000000000000190"
-"000000030000000000000191"
-"000000030000000000000192"
-"000000030000000000000193"
-"000000030000000000000194"
-"000000030000000000000195"
-"000000030000000000000196"
-"000000030000000000000197"
-"000000030000000000000198"
-"000000030000000000000199"
-"00000003000000000000019A"
-"00000003000000000000019B"
-"00000003000000000000019C"
-"00000003000000000000019D"
-"00000003000000000000019E"
-"00000003000000000000019F"
-"0000000300000000000001A0"
-"0000000300000000000001A1"
-"0000000300000000000001A2"
-"0000000300000000000001A3"
-"0000000300000000000001A4"
-"0000000300000000000001A5"
-"0000000300000000000001A6"
-"0000000300000000000001A7"
-"0000000300000000000001A8"
-"0000000300000000000001A9"
-"0000000300000000000001AA"
-"0000000300000000000001AB"
-"0000000300000000000001AC"
-"0000000300000000000001AD"
-"0000000300000000000001AE"
-"0000000300000000000001AF"
-"0000000300000000000001B0"
-"0000000300000000000001B1"
-"0000000300000000000001B2"
-"0000000300000000000001B3"
-"0000000300000000000001B4"
-"0000000300000000000001B5"
-"0000000300000000000001B6"
-"0000000300000000000001B7"
-"0000000300000000000001B8"
-"0000000300000000000001B9"
-"0000000300000000000001BA"
-"0000000300000000000001BB"
-"0000000300000000000001BC"
-"0000000300000000000001BD"
-"0000000300000000000001BE"
-"0000000300000000000001BF"
-"0000000300000000000001C0"
-"0000000300000000000001C1"
-"0000000300000000000001C2"
-"0000000300000000000001C3"
-"0000000300000000000001C4"
-"0000000300000000000001C5"
-"0000000300000000000001C6"
-"0000000300000000000001C7"
-"0000000300000000000001C8"
-"0000000300000000000001C9"
-"0000000300000000000001CA"
-"0000000300000000000001CB"
-"0000000300000000000001CC"
-"0000000300000000000001CD"
-"0000000300000000000001CE"
-"0000000300000000000001CF"
-"0000000300000000000001D0"
-"0000000300000000000001D1"
-"0000000300000000000001D2"
-"0000000300000000000001D3"
-"0000000300000000000001D4"
-"0000000300000000000001D5"
-"0000000300000000000001D6"
-"0000000300000000000001D7"
-"0000000300000000000001D8"
-"0000000300000000000001D9"
-"0000000300000000000001DA"
-"0000000300000000000001DB"
-"0000000300000000000001DC"
-"0000000300000000000001DD"
-"0000000300000000000001DE"
-"0000000300000000000001DF"
-"0000000300000000000001E0"
-"0000000300000000000001E1"
-"0000000300000000000001E2"
-"0000000300000000000001E3"
-"0000000300000000000001E4"
-"0000000300000000000001E5"
-"0000000300000000000001E6"
-"0000000300000000000001E7"
-"0000000300000000000001E8"
-"0000000300000000000001E9"
-"0000000300000000000001EA"
-"0000000300000000000001EB"
-"0000000300000000000001EC"
-"0000000300000000000001ED"
-"0000000300000000000001EE"
-"0000000300000000000001EF"
-"0000000300000000000001F0"
-"0000000300000000000001F1"
-"0000000300000000000001F2"
-"0000000300000000000001F3"
-"0000000300000000000001F4"
-"0000000300000000000001F5"
-"0000000300000000000001F6"
-"0000000300000000000001F7"
-"0000000300000000000001F8"
-"0000000300000000000001F9"
-"0000000300000000000001FA"
-"0000000300000000000001FB"
-"0000000300000000000001FC"
-"0000000300000000000001FD"
-"0000000300000000000001FE"
-"0000000300000000000001FF"
-"000000030000000000000200"
-"000000030000000000000201"
-"000000030000000000000202"
-"000000030000000000000203"
-"000000030000000000000204"
-"000000030000000000000205"
-"000000030000000000000206"
-"000000030000000000000207"
-"000000030000000000000208"
-"000000030000000000000209"
-"00000003000000000000020A"
-"00000003000000000000020B"
-"00000003000000000000020C"
-"00000003000000000000020D"
-"00000003000000000000020E"
-"00000003000000000000020F"
-"000000030000000000000210"
-"000000030000000000000211"
-"000000030000000000000212"
-"000000030000000000000213"
-"000000030000000000000214"
-"000000030000000000000215"
-"000000030000000000000216"
-"000000030000000000000217"
-"000000030000000000000218"
-"000000030000000000000219"
-"00000003000000000000021A"
-"00000003000000000000021B"
-"00000003000000000000021C"
-"00000003000000000000021D"
-"00000003000000000000021E"
-"00000003000000000000021F"
-"000000030000000000000220"
-"000000030000000000000221"
-"000000030000000000000222"
-"000000030000000000000223"
-"000000030000000000000224"
-"000000030000000000000225"
-"000000030000000000000226"
-"000000030000000000000227"
-"000000030000000000000228"
-"000000030000000000000229"
-"00000003000000000000022A"
-"00000003000000000000022B"
-"00000003000000000000022C"
-"00000003000000000000022D"
-"00000003000000000000022E"
-"00000003000000000000022F"
-"000000030000000000000230"
-"000000030000000000000231"
-"000000030000000000000232"
-"000000030000000000000233"
-"000000030000000000000234"
-"000000030000000000000235"
-"000000030000000000000236"
-"000000030000000000000237"
-"000000030000000000000238"
-"000000030000000000000239"
-"00000003000000000000023A"
-"00000003000000000000023B"
-"00000003000000000000023C"
-"00000003000000000000023D"
-"00000003000000000000023E"
-"00000003000000000000023F"
-"000000030000000000000240"
-"000000030000000000000241"
-"000000030000000000000242"
-"000000030000000000000243"
-"000000030000000000000244"
-"000000030000000000000245"
-"000000030000000000000246"
-"000000030000000000000247"
-"000000030000000000000248"
-"000000030000000000000249"
-"00000003000000000000024A"
-"00000003000000000000024B"
-"00000003000000000000024C"
-"00000003000000000000024D"
-"00000003000000000000024E"
-"00000003000000000000024F"
-"000000030000000000000250"
-"000000030000000000000251"
-"000000030000000000000252"
-"000000030000000000000253"
-"000000030000000000000254"
-"000000030000000000000255"
-"000000030000000000000256"
-"000000030000000000000257"
-"000000030000000000000258"
-"000000030000000000000259"
-"00000003000000000000025A"
-"00000003000000000000025B"
-"00000003000000000000025C"
-"00000003000000000000025D"
-"00000003000000000000025E"
-"00000003000000000000025F"
-"000000030000000000000260"
-"000000030000000000000261"
-"000000030000000000000262"
-"000000030000000000000263"
-"000000030000000000000264"
-"000000030000000000000265"
-"000000030000000000000266"
-"000000030000000000000267"
-"000000030000000000000268"
-"000000030000000000000269"
-"00000003000000000000026A"
-"00000003000000000000026B"
-"00000003000000000000026C"
-"00000003000000000000026D"
-"00000003000000000000026E"
-"00000003000000000000026F"
-"000000030000000000000270"
-"000000030000000000000271"
-"000000030000000000000272"
-"000000030000000000000273"
-"000000030000000000000274"
-"000000030000000000000275"
-"000000030000000000000276"
-"000000030000000000000277"
-"000000030000000000000278"
-"000000030000000000000279"
-"00000003000000000000027A"
-"00000003000000000000027B"
-"00000003000000000000027C"
-"00000003000000000000027D"
-"00000003000000000000027E"
-"00000003000000000000027F"
-"000000030000000000000280"
-"000000030000000000000281"
-"000000030000000000000282"
-"000000030000000000000283"
-"000000030000000000000284"
-"000000030000000000000285"
-"000000030000000000000286"
-"000000030000000000000287"
-"000000030000000000000288"
-"000000030000000000000289"
-"00000003000000000000028A"
-"00000003000000000000028B"
-"00000003000000000000028C"
-"00000003000000000000028D"
-"00000003000000000000028E"
-"00000003000000000000028F"
-"000000030000000000000290"
-"000000030000000000000291"
-"000000030000000000000292"
-"000000030000000000000293"
-"000000030000000000000294"
-"000000030000000000000295"
-"000000030000000000000296"
-"000000030000000000000297"
-"000000030000000000000298"
-"000000030000000000000299"
-"00000003000000000000029A"
-"00000003000000000000029B"
-"00000003000000000000029C"
-"00000003000000000000029D"
-"00000003000000000000029E"
-"00000003000000000000029F"
-"0000000300000000000002A0"
-"0000000300000000000002A1"
-"0000000300000000000002A2"
-"0000000300000000000002A3"
-"0000000300000000000002A4"
-"0000000300000000000002A5"
-"0000000300000000000002A6"
-"0000000300000000000002A7"
-"0000000300000000000002A8"
-"0000000300000000000002A9"
-"0000000300000000000002AA"
-"0000000300000000000002AB"
-"0000000300000000000002AC"
-"0000000300000000000002AD"
-"0000000300000000000002AE"
-"0000000300000000000002AF"
-"0000000300000000000002B0"
-"0000000300000000000002B1"
-"0000000300000000000002B2"
-"0000000300000000000002B3"
-"0000000300000000000002B4"
-"0000000300000000000002B5"
-"0000000300000000000002B6"
-"0000000300000000000002B7"
-"0000000300000000000002B8"
-"0000000300000000000002B9"
-"0000000300000000000002BA"
-"0000000300000000000002BB"
-"0000000300000000000002BC"
-"0000000300000000000002BD"
-"0000000300000000000002BE"
-"0000000300000000000002BF"
-"0000000300000000000002C0"
-"0000000300000000000002C1"
-"0000000300000000000002C2"
-"0000000300000000000002C3"
-"0000000300000000000002C4"
-"0000000300000000000002C5"
-"0000000300000000000002C6"
-"0000000300000000000002C7"
-"0000000300000000000002C8"
-"0000000300000000000002C9"
-"0000000300000000000002CA"
-"0000000300000000000002CB"
-"0000000300000000000002CC"
-"0000000300000000000002CD"
-"0000000300000000000002CE"
-"0000000300000000000002CF"
-"0000000300000000000002D0"
-"0000000300000000000002D1"
-"0000000300000000000002D2"
-"0000000300000000000002D3"
-"0000000300000000000002D4"
-"0000000300000000000002D5"
-"0000000300000000000002D6"
-"0000000300000000000002D7"
-"0000000300000000000002D8"
-"0000000300000000000002D9"
-"0000000300000000000002DA"
-"0000000300000000000002DB"
-"0000000300000000000002DC"
-"0000000300000000000002DD"
-"0000000300000000000002DE"
-"0000000300000000000002DF"
-"0000000300000000000002E0"
-"0000000300000000000002E1"
-"0000000300000000000002E2"
-"0000000300000000000002E3"
-"0000000300000000000002E4"
-"0000000300000000000002E5"
-"0000000300000000000002E6"
-"0000000300000000000002E7"
-"0000000300000000000002E8"
-"0000000300000000000002E9"
-"0000000300000000000002EA"
-"0000000300000000000002EB"
-"0000000300000000000002EC"
-"0000000300000000000002ED"
-"0000000300000000000002EE"
-"0000000300000000000002EF"
-"0000000300000000000002F0"
-"0000000300000000000002F1"
-"0000000300000000000002F2"
-"0000000300000000000002F3"
-"0000000300000000000002F4"
-"0000000300000000000002F5"
-"0000000300000000000002F6"
-"0000000300000000000002F7"
-"0000000300000000000002F8"
-"0000000300000000000002F9"
-"0000000300000000000002FA"
-"0000000300000000000002FB"
-"0000000300000000000002FC"
-"0000000300000000000002FD"
-"0000000300000000000002FE"
-"0000000300000000000002FF"
-"000000030000000000000300"
-"000000030000000000000301"
-"000000030000000000000302"
-"000000030000000000000303"
-"000000030000000000000304"
-"000000030000000000000305"
-"000000030000000000000306"
-"000000030000000000000307"
-"000000030000000000000308"
-"000000030000000000000309"
-"00000003000000000000030A"
-"00000003000000000000030B"
-"00000003000000000000030C"
-"00000003000000000000030D"
-"00000003000000000000030E"
-"00000003000000000000030F"
-"000000030000000000000310"
-"000000030000000000000311"
-"000000030000000000000312"
-"000000030000000000000313"
-"000000030000000000000314"
-"000000030000000000000315"
-"000000030000000000000316"
-"000000030000000000000317"
-"000000030000000000000318"
-"000000030000000000000319"
-"00000003000000000000031A"
-"00000003000000000000031B"
-"00000003000000000000031C"
-"00000003000000000000031D"
-"00000003000000000000031E"
-"00000003000000000000031F"
-"000000030000000000000320"
-"000000030000000000000321"
-"000000030000000000000322"
-"000000030000000000000323"
-"000000030000000000000324"
-"000000030000000000000325"
-"000000030000000000000326"
-"000000030000000000000327"
-"000000030000000000000328"
-"000000030000000000000329"
-"00000003000000000000032A"
-"00000003000000000000032B"
-"00000003000000000000032C"
-"00000003000000000000032D"
-"00000003000000000000032E"
-"00000003000000000000032F"
-"000000030000000000000330"
-"000000030000000000000331"
-"000000030000000000000332"
-"000000030000000000000333"
-"000000030000000000000334"
-"000000030000000000000335"
-"000000030000000000000336"
-"000000030000000000000337"
-"000000030000000000000338"
-"000000030000000000000339"
-"00000003000000000000033A"
-"00000003000000000000033B"
-"00000003000000000000033C"
-"00000003000000000000033D"
-"00000003000000000000033E"
-"00000003000000000000033F"
-"000000030000000000000340"
-"000000030000000000000341"
-"000000030000000000000342"
-"000000030000000000000343"
-"000000030000000000000344"
-"000000030000000000000345"
-"000000030000000000000346"
-"000000030000000000000347"
-"000000030000000000000348"
-"000000030000000000000349"
-"00000003000000000000034A"
-"00000003000000000000034B"
-"00000003000000000000034C"
-"00000003000000000000034D"
-"00000003000000000000034E"
-"00000003000000000000034F"
-"000000030000000000000350"
-"000000030000000000000351"
-"000000030000000000000352"
-"000000030000000000000353"
-"000000030000000000000354"
-"000000030000000000000355"
-"000000030000000000000356"
-"000000030000000000000357"
-"000000030000000000000358"
-"000000030000000000000359"
-"00000003000000000000035A"
-"00000003000000000000035B"
-"00000003000000000000035C"
-"00000003000000000000035D"
-"00000003000000000000035E"
-"00000003000000000000035F"
-"000000030000000000000360"
-"000000030000000000000361"
-"000000030000000000000362"
-"000000030000000000000363"
-"000000030000000000000364"
-"000000030000000000000365"
-"000000030000000000000366"
-"000000030000000000000367"
-"000000030000000000000368"
-"000000030000000000000369"
-"00000003000000000000036A"
-"00000003000000000000036B"
-"00000003000000000000036C"
-"00000003000000000000036D"
-"00000003000000000000036E"
-"00000003000000000000036F"
-"000000030000000000000370"
-"000000030000000000000371"
-"000000030000000000000372"
-"000000030000000000000373"
-"000000030000000000000374"
-"000000030000000000000375"
-"000000030000000000000376"
-"000000030000000000000377"
-"000000030000000000000378"
-"000000030000000000000379"
-"00000003000000000000037A"
-"00000003000000000000037B"
-"00000003000000000000037C"
-"00000003000000000000037D"
-"00000003000000000000037E"
-"00000003000000000000037F"
-"000000030000000000000380"
-"000000030000000000000381"
-"000000030000000000000382"
-"000000030000000000000383"
-"000000030000000000000384"
-"000000030000000000000385"
-"000000030000000000000386"
-"000000030000000000000387"
-"000000030000000000000388"
-"000000030000000000000389"
-"00000003000000000000038A"
-"00000003000000000000038B"
-"00000003000000000000038C"
-"00000003000000000000038D"
-"00000003000000000000038E"
-"00000003000000000000038F"
-"000000030000000000000390"
-"000000030000000000000391"
-"000000030000000000000392"
-"000000030000000000000393"
-"000000030000000000000394"
-"000000030000000000000395"
-"000000030000000000000396"
-"000000030000000000000397"
-"000000030000000000000398"
-"000000030000000000000399"
-"00000003000000000000039A"
-"00000003000000000000039B"
-"00000003000000000000039C"
-"00000003000000000000039D"
-"00000003000000000000039E"
-"00000003000000000000039F"
-"0000000300000000000003A0"
-"0000000300000000000003A1"
-"0000000300000000000003A2"
-"0000000300000000000003A3"
-"0000000300000000000003A4"
-"0000000300000000000003A5"
-"0000000300000000000003A6"
-"0000000300000000000003A7"
-"0000000300000000000003A8"
-"0000000300000000000003A9"
-"0000000300000000000003AA"
-"0000000300000000000003AB"
-"0000000300000000000003AC"
-"0000000300000000000003AD"
-"0000000300000000000003AE"
-"0000000300000000000003AF"
-"0000000300000000000003B0"
-"0000000300000000000003B1"
-"0000000300000000000003B2"
-"0000000300000000000003B3"
-"0000000300000000000003B4"
-"0000000300000000000003B5"
-"0000000300000000000003B6"
-"0000000300000000000003B7"
-"0000000300000000000003B8"
-"0000000300000000000003B9"
-"0000000300000000000003BA"
-"0000000300000000000003BB"
-"0000000300000000000003BC"
-"0000000300000000000003BD"
-"0000000300000000000003BE"
-"0000000300000000000003BF"
-"0000000300000000000003C0"
-"0000000300000000000003C1"
-"0000000300000000000003C2"
-"0000000300000000000003C3"
-"0000000300000000000003C4"
-"0000000300000000000003C5"
-"0000000300000000000003C6"
-"0000000300000000000003C7"
-"0000000300000000000003C8"
-"0000000300000000000003C9"
-"0000000300000000000003CA"
-"0000000300000000000003CB"
-"0000000300000000000003CC"
-"0000000300000000000003CD"
-"0000000300000000000003CE"
-"0000000300000000000003CF"
-"0000000300000000000003D0"
-"0000000300000000000003D1"
-"0000000300000000000003D2"
-"0000000300000000000003D3"
-"0000000300000000000003D4"
-"0000000300000000000003D5"
-"0000000300000000000003D6"
-"0000000300000000000003D7"
-"0000000300000000000003D8"
-"0000000300000000000003D9"
-"0000000300000000000003DA"
-"0000000300000000000003DB"
-"0000000300000000000003DC"
-"0000000300000000000003DD"
-"0000000300000000000003DE"
-"0000000300000000000003DF"
-"0000000300000000000003E0"
-"0000000300000000000003E1"
-"0000000300000000000003E2"
-"0000000300000000000003E3"
-"0000000300000000000003E4"
-"0000000300000000000003E5"
-"0000000300000000000003E6"
-"0000000300000000000003E7"
-"0000000300000000000003E8"
-"0000000300000000000003E9"
-"0000000300000000000003EA"
-"0000000300000000000003EB"
-"0000000300000000000003EC"
-"0000000300000000000003ED"
-"0000000300000000000003EE"
-"0000000300000000000003EF"
-"0000000300000000000003F0"
-"0000000300000000000003F1"
-"0000000300000000000003F2"
-"0000000300000000000003F3"
-"0000000300000000000003F4"
-"0000000300000000000003F5"
-"0000000300000000000003F6"
-"0000000300000000000003F7"
-"0000000300000000000003F8"
-"0000000300000000000003F9"
-"0000000300000000000003FA"
-"0000000300000000000003FB"
-"0000000300000000000003FC"
-"0000000300000000000003FD"
-"0000000300000000000003FE"
-"0000000300000000000003FF"
-"000000030000000000000400"
-"000000030000000000000401"
-"000000030000000000000402"
-"000000030000000000000403"
-"000000030000000000000404"
-"000000030000000000000405"
-"000000030000000000000406"
-"000000030000000000000407"
-"000000030000000000000408"
-"000000030000000000000409"
-"00000003000000000000040A"
-"00000003000000000000040B"
-"00000003000000000000040C"
-"00000003000000000000040D"
-"00000003000000000000040E"
-"00000003000000000000040F"
-"000000030000000000000410"
-"000000030000000000000411"
-"000000030000000000000412"
-"000000030000000000000413"
-"000000030000000000000414"
-"000000030000000000000415"
-"000000030000000000000416"
-"000000030000000000000417"
-"000000030000000000000418"
-"000000030000000000000419"
-"00000003000000000000041A"
-"00000003000000000000041B"
-"00000003000000000000041C"
-"00000003000000000000041D"
-"00000003000000000000041E"
-"00000003000000000000041F"
-"000000030000000000000420"
-"000000030000000000000421"
-"000000030000000000000422"
-"000000030000000000000423"
-"000000030000000000000424"
-"000000030000000000000425"
-"000000030000000000000426"
-"000000030000000000000427"
-"000000030000000000000428"
-"000000030000000000000429"
-"00000003000000000000042A"
-"00000003000000000000042B"
-"00000003000000000000042C"
-"00000003000000000000042D"
-"00000003000000000000042E"
-"00000003000000000000042F"
-"000000030000000000000430"
-"000000030000000000000431"
-"000000030000000000000432"
-"000000030000000000000433"
-"000000030000000000000434"
-"000000030000000000000435"
-"000000030000000000000436"
-"000000030000000000000437"
-"000000030000000000000438"
-"000000030000000000000439"
-"00000003000000000000043A"
-"00000003000000000000043B"
-"00000003000000000000043C"
-"00000003000000000000043D"
-"00000003000000000000043E"
-"00000003000000000000043F"
-"000000030000000000000440"
-"000000030000000000000441"
-"000000030000000000000442"
-"000000030000000000000443"
-"000000030000000000000444"
-"000000030000000000000445"
-"000000030000000000000446"
-"000000030000000000000447"
-"000000030000000000000448"
-"000000030000000000000449"
-"00000003000000000000044A"
-"00000003000000000000044B"
-"00000003000000000000044C"
-"00000003000000000000044D"
-"00000003000000000000044E"
-"00000003000000000000044F"
-"000000030000000000000450"
-"000000030000000000000451"
-"000000030000000000000452"
-"000000030000000000000453"
-"000000030000000000000454"
-"000000030000000000000455"
-"000000030000000000000456"
-"000000030000000000000457"
-"000000030000000000000458"
-"000000030000000000000459"
-"00000003000000000000045A"
-"00000003000000000000045B"
-"00000003000000000000045C"
-"00000003000000000000045D"
-"00000003000000000000045E"
-"00000003000000000000045F"
-"000000030000000000000460"
-"000000030000000000000461"
-"000000030000000000000462"
-"000000030000000000000463"
-"000000030000000000000464"
-"000000030000000000000465"
-"000000030000000000000466"
-"000000030000000000000467"
-"000000030000000000000468"
-"000000030000000000000469"
-"00000003000000000000046A"
-"00000003000000000000046B"
-"00000003000000000000046C"
-"00000003000000000000046D"
-"00000003000000000000046E"
-"00000003000000000000046F"
-"000000030000000000000470"
-"000000030000000000000471"
-"000000030000000000000472"
-"000000030000000000000473"
-"000000030000000000000474"
-"000000030000000000000475"
-"000000030000000000000476"
-"000000030000000000000477"
-"000000030000000000000478"
-"000000030000000000000479"
-"00000003000000000000047A"
-"00000003000000000000047B"
-"00000003000000000000047C"
-"00000003000000000000047D"
-"00000003000000000000047E"
-"00000003000000000000047F"
-"000000030000000000000480"
-"000000030000000000000481"
-"000000030000000000000482"
-"000000030000000000000483"
-"000000030000000000000484"
-"000000030000000000000485"
-"000000030000000000000486"
-"000000030000000000000487"
-"000000030000000000000488"
-"000000030000000000000489"
-"00000003000000000000048A"
-"00000003000000000000048B"
-"00000003000000000000048C"
-"00000003000000000000048D"
-"00000003000000000000048E"
-"00000003000000000000048F"
-"000000030000000000000490"
-"000000030000000000000491"
-"000000030000000000000492"
-"000000030000000000000493"
-"000000030000000000000494"
-"000000030000000000000495"
-"000000030000000000000496"
-"000000030000000000000497"
-"000000030000000000000498"
-"000000030000000000000499"
-"00000003000000000000049A"
-"00000003000000000000049B"
-"00000003000000000000049C"
-"00000003000000000000049D"
-"00000003000000000000049E"
-"00000003000000000000049F"
-"0000000300000000000004A0"
-"0000000300000000000004A1"
-"0000000300000000000004A2"
-"0000000300000000000004A3"
-"0000000300000000000004A4"
-"0000000300000000000004A5"
-"0000000300000000000004A6"
-"0000000300000000000004A7"
-"0000000300000000000004A8"
-"0000000300000000000004A9"
-"0000000300000000000004AA"
-"0000000300000000000004AB"
-"0000000300000000000004AC"
-"0000000300000000000004AD"
-"0000000300000000000004AE"
-"0000000300000000000004AF"
-"0000000300000000000004B0"
-"0000000300000000000004B1"
-"0000000300000000000004B2"
-"0000000300000000000004B3"
-"0000000300000000000004B4"
-"0000000300000000000004B5"
-"0000000300000000000004B6"
-"0000000300000000000004B7"
-"0000000300000000000004B8"
-"0000000300000000000004B9"
-"0000000300000000000004BA"
-"0000000300000000000004BB"
-"0000000300000000000004BC"
-"0000000300000000000004BD"
-"0000000300000000000004BE"
-"0000000300000000000004BF"
-"0000000300000000000004C0"
-"0000000300000000000004C1"
-"0000000300000000000004C2"
-"0000000300000000000004C3"
-"0000000300000000000004C4"
-"0000000300000000000004C5"
-"0000000300000000000004C6"
-"0000000300000000000004C7"
-"0000000300000000000004C8"
-"0000000300000000000004C9"
-"0000000300000000000004CA"
-"0000000300000000000004CB"
-"0000000300000000000004CC"
-"0000000300000000000004CD"
-"0000000300000000000004CE"
-"0000000300000000000004CF"
-"0000000300000000000004D0"
-"0000000300000000000004D1"
-"0000000300000000000004D2"
-"0000000300000000000004D3"
-"0000000300000000000004D4"
-"0000000300000000000004D5"
-"0000000300000000000004D6"
-"0000000300000000000004D7"
-"0000000300000000000004D8"
-"0000000300000000000004D9"
-"0000000300000000000004DA"
-"0000000300000000000004DB"
-"0000000300000000000004DC"
-"0000000300000000000004DD"
-"0000000300000000000004DE"
-"0000000300000000000004DF"
-"0000000300000000000004E0"
-"0000000300000000000004E1"
-"0000000300000000000004E2"
-"0000000300000000000004E3"
-"0000000300000000000004E4"
-"0000000300000000000004E5"
-"0000000300000000000004E6"
-"0000000300000000000004E7"
-"0000000300000000000004E8"
-"0000000300000000000004E9"
-"0000000300000000000004EA"
-"0000000300000000000004EB"
-"0000000300000000000004EC"
-"0000000300000000000004ED"
-"0000000300000000000004EE"
-"0000000300000000000004EF"
-"0000000300000000000004F0"
-"0000000300000000000004F1"
-"0000000300000000000004F2"
-"0000000300000000000004F3"
-"0000000300000000000004F4"
-"0000000300000000000004F5"
-"0000000300000000000004F6"
-"0000000300000000000004F7"
-"0000000300000000000004F8"
-"0000000300000000000004F9"
-"0000000300000000000004FA"
-"0000000300000000000004FB"
-"0000000300000000000004FC"
-"0000000300000000000004FD"
-"0000000300000000000004FE"
-"0000000300000000000004FF"
-"000000030000000000000500"
-"000000030000000000000501"
-"000000030000000000000502"
-"000000030000000000000503"
-"000000030000000000000504"
-"000000030000000000000505"
-"000000030000000000000506"
-"000000030000000000000507"
-"000000030000000000000508"
-"000000030000000000000509"
-"00000003000000000000050A"
-"00000003000000000000050B"
-"00000003000000000000050C"
-"00000003000000000000050D"
-"00000003000000000000050E"
-"00000003000000000000050F"
-"000000030000000000000510"
-"000000030000000000000511"
-"000000030000000000000512"
-"000000030000000000000513"
-"000000030000000000000514"
-"000000030000000000000515"
-"000000030000000000000516"
-"000000030000000000000517"
-"000000030000000000000518"
-"000000030000000000000519"
-"00000003000000000000051A"
-"00000003000000000000051B"
-"00000003000000000000051C"
-"00000003000000000000051D"
-"00000003000000000000051E"
-"00000003000000000000051F"
-"000000030000000000000520"
-"000000030000000000000521"
-"000000030000000000000522"
-"000000030000000000000523"
-"000000030000000000000524"
-"000000030000000000000525"
-"000000030000000000000526"
-"000000030000000000000527"
-"000000030000000000000528"
-"000000030000000000000529"
-"00000003000000000000052A"
-"00000003000000000000052B"
-"00000003000000000000052C"
-"00000003000000000000052D"
-"00000003000000000000052E"
-"00000003000000000000052F"
-"000000030000000000000530"
-"000000030000000000000531"
-"000000030000000000000532"
-"000000030000000000000533"
-"000000030000000000000534"
-"000000030000000000000535"
-"000000030000000000000536"
-"000000030000000000000537"
-"000000030000000000000538"
-"000000030000000000000539"
-"00000003000000000000053A"
-"00000003000000000000053B"
-"00000003000000000000053C"
-"00000003000000000000053D"
-"00000003000000000000053E"
-"00000003000000000000053F"
-"000000030000000000000540"
-"000000030000000000000541"
-"000000030000000000000542"
-"000000030000000000000543"
-"000000030000000000000544"
-"000000030000000000000545"
-"000000030000000000000546"
-"000000030000000000000547"
-"000000030000000000000548"
-"000000030000000000000549"
-"00000003000000000000054A"
-"00000003000000000000054B"
-"00000003000000000000054C"
-"00000003000000000000054D"
-"00000003000000000000054E"
-"00000003000000000000054F"
-"000000030000000000000550"
-"000000030000000000000551"
-"000000030000000000000552"
-"000000030000000000000553"
-"000000030000000000000554"
-"000000030000000000000555"
-"000000030000000000000556"
-"000000030000000000000557"
-"000000030000000000000558"
-"000000030000000000000559"
-"00000003000000000000055A"
-"00000003000000000000055B"
-"00000003000000000000055C"
-"00000003000000000000055D"
-"00000003000000000000055E"
-"00000003000000000000055F"
-"000000030000000000000560"
-"000000030000000000000561"
-"000000030000000000000562"
-"000000030000000000000563"
-"000000030000000000000564"
-"000000030000000000000565"
-"000000030000000000000566"
-"000000030000000000000567"
-"000000030000000000000568"
-"000000030000000000000569"
-"00000003000000000000056A"
-"00000003000000000000056B"
-"00000003000000000000056C"
-"00000003000000000000056D"
-"00000003000000000000056E"
-"00000003000000000000056F"
-"000000030000000000000570"
-"000000030000000000000571"
-"000000030000000000000572"
-"000000030000000000000573"
-"000000030000000000000574"
-"000000030000000000000575"
-"000000030000000000000576"
-"000000030000000000000577"
-"000000030000000000000578"
-"000000030000000000000579"
-"00000003000000000000057A"
-"00000003000000000000057B"
-"00000003000000000000057C"
-"00000003000000000000057D"
-"00000003000000000000057E"
-"00000003000000000000057F"
-"000000030000000000000580"
-"000000030000000000000581"
-"000000030000000000000582"
-"000000030000000000000583"
-"000000030000000000000584"
-"000000030000000000000585"
-"000000030000000000000586"
-"000000030000000000000587"
-"000000030000000000000588"
-"000000030000000000000589"
-"00000003000000000000058A"
-"00000003000000000000058B"
-"00000003000000000000058C"
-"00000003000000000000058D"
-"00000003000000000000058E"
-"00000003000000000000058F"
-"000000030000000000000590"
-"000000030000000000000591"
-"000000030000000000000592"
-"000000030000000000000593"
-"000000030000000000000594"
-"000000030000000000000595"
-"000000030000000000000596"
-"000000030000000000000597"
-"000000030000000000000598"
-"000000030000000000000599"
-"00000003000000000000059A"
-"00000003000000000000059B"
-"00000003000000000000059C"
-"00000003000000000000059D"
-"00000003000000000000059E"
-"00000003000000000000059F"
-"0000000300000000000005A0"
-"0000000300000000000005A1"
-"0000000300000000000005A2"
-"0000000300000000000005A3"
-"0000000300000000000005A4"
-"0000000300000000000005A5"
-"0000000300000000000005A6"
-"0000000300000000000005A7"
-"0000000300000000000005A8"
-"0000000300000000000005A9"
-"0000000300000000000005AA"
-"0000000300000000000005AB"
-"0000000300000000000005AC"
-"0000000300000000000005AD"
-"0000000300000000000005AE"
-"0000000300000000000005AF"
-"0000000300000000000005B0"
-"0000000300000000000005B1"
-"0000000300000000000005B2"
-"0000000300000000000005B3"
-"0000000300000000000005B4"
-"0000000300000000000005B5"
-"0000000300000000000005B6"
-"0000000300000000000005B7"
-"0000000300000000000005B8"
-"0000000300000000000005B9"
-"0000000300000000000005BA"
-"0000000300000000000005BB"
-"0000000300000000000005BC"
-"0000000300000000000005BD"
-"0000000300000000000005BE"
-"0000000300000000000005BF"
-"0000000300000000000005C0"
-"0000000300000000000005C1"
-"0000000300000000000005C2"
-"0000000300000000000005C3"
-"0000000300000000000005C4"
-"0000000300000000000005C5"
-"0000000300000000000005C6"
-"0000000300000000000005C7"
-"0000000300000000000005C8"
-"0000000300000000000005C9"
-"0000000300000000000005CA"
-"0000000300000000000005CB"
-"0000000300000000000005CC"
-"0000000300000000000005CD"
-"0000000300000000000005CE"
-"0000000300000000000005CF"
-"0000000300000000000005D0"
-"0000000300000000000005D1"
-"0000000300000000000005D2"
-"0000000300000000000005D3"
-"0000000300000000000005D4"
-"0000000300000000000005D5"
-"0000000300000000000005D6"
-"0000000300000000000005D7"
-"0000000300000000000005D8"
-"0000000300000000000005D9"
-"0000000300000000000005DA"
-"0000000300000000000005DB"
-"0000000300000000000005DC"
-"0000000300000000000005DD"
-"0000000300000000000005DE"
-"0000000300000000000005DF"
-"0000000300000000000005E0"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm
new file mode 100644
index 0000000..f63d51b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/in_let/in_let.8.adm
@@ -0,0 +1,2 @@
+{ "cid": "c1", "pid": "p2", "ts": 2000 }
+{ "cid": "c2", "pid": "p2", "ts": 2100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 851355a..7050d49 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -13676,11 +13676,6 @@
<output-dir compare="Text">length</output-dir>
</compilation-unit>
</test-case>
- <test-case FilePath="binary">
- <compilation-unit name="query_id">
- <output-dir compare="Text">query_id</output-dir>
- </compilation-unit>
- </test-case>
</test-group>
<test-group name="unnest">
<test-case FilePath="unnest">
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 63573a1..aac01f9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1657,7 +1657,7 @@ public class BuiltinFunctions {
addFunction(CREATE_RECTANGLE, ARectangleTypeComputer.INSTANCE, true);
addFunction(CREATE_UUID, AUUIDTypeComputer.INSTANCE, false);
addFunction(UUID, AUUIDTypeComputer.INSTANCE, false);
- addFunction(CREATE_QUERY_UID, ABinaryTypeComputer.INSTANCE, false);
+ addPrivateFunction(CREATE_QUERY_UID, ABinaryTypeComputer.INSTANCE, false);
addFunction(UUID_CONSTRUCTOR, AUUIDTypeComputer.INSTANCE, true);
addFunction(RANDOM, ADoubleTypeComputer.INSTANCE, false);
addFunction(RANDOM_WITH_SEED, NumericUnaryTypeComputer.INSTANCE_DOUBLE, false);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
index adb8874..86d0596 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CreateQueryUIDDescriptor.java
@@ -19,15 +19,14 @@
package org.apache.asterix.runtime.evaluators.functions;
import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.runningaggregates.base.AbstractRunningAggregateFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
@@ -35,31 +34,26 @@ import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
/**
* Create global unique id within a query.
*/
-public class CreateQueryUIDDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+public class CreateQueryUIDDescriptor extends AbstractRunningAggregateFunctionDynamicDescriptor {
private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new CreateQueryUIDDescriptor();
- }
- };
+ public static final IFunctionDescriptorFactory FACTORY = CreateQueryUIDDescriptor::new;
@Override
- public IScalarEvaluatorFactory createEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
+ public IRunningAggregateEvaluatorFactory createRunningAggregateEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ return new IRunningAggregateEvaluatorFactory() {
private static final long serialVersionUID = 1L;
private static final int BINARY_LENGTH = 14;
private static final int PAYLOAD_START = 2;
@Override
- public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator(IEvaluatorContext ctx) {
// Format: |TypeTag | PayloadLength | Payload |
// TypeTag: 1 byte
// PayloadLength: 1 byte
// Payload: 12 bytes: |partition-id (4 bytes) | local-id (8 bytes) |
- byte[] uidBytes = new byte[BINARY_LENGTH];
+ final byte[] uidBytes = new byte[BINARY_LENGTH];
// Writes the type tag.
uidBytes[0] = ATypeTag.SERIALIZED_BINARY_TYPE_TAG;
// Writes the payload size.
@@ -68,10 +62,13 @@ public class CreateQueryUIDDescriptor extends AbstractScalarFunctionDynamicDescr
IntegerPointable.setInteger(uidBytes, PAYLOAD_START,
ctx.getTaskContext().getTaskAttemptId().getTaskId().getPartition());
- return new IScalarEvaluator() {
+ return new IRunningAggregateEvaluator() {
+ @Override
+ public void init() {
+ }
@Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ public void step(IFrameTupleReference tuple, IPointable result) {
// Increments the Unique ID value.
for (int i = BINARY_LENGTH - 1; i >= PAYLOAD_START; i--) {
if (++uidBytes[i] != 0) {
@@ -89,5 +86,4 @@ public class CreateQueryUIDDescriptor extends AbstractScalarFunctionDynamicDescr
public FunctionIdentifier getIdentifier() {
return BuiltinFunctions.CREATE_QUERY_UID;
}
-
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
index 313fc30..56040dc 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -36,6 +36,11 @@ public class RunningAggregateOperator extends AbstractAssignOperator {
super(variables, expressions);
}
+ public RunningAggregateOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
+ this.variables.add(var);
+ this.expressions.add(expr);
+ }
+
@Override
public LogicalOperatorTag getOperatorTag() {
return LogicalOperatorTag.RUNNINGAGGREGATE;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index aa2ecdc..0df9851 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -487,11 +487,16 @@ public class OperatorManipulationUtil {
public static List<Mutable<ILogicalExpression>> createVariableReferences(Collection<LogicalVariable> varList,
SourceLocation sourceLoc) {
List<Mutable<ILogicalExpression>> varRefs = new ArrayList<>(varList.size());
+ createVariableReferences(varList, sourceLoc, varRefs);
+ return varRefs;
+ }
+
+ public static void createVariableReferences(Collection<LogicalVariable> varList, SourceLocation sourceLoc,
+ List<Mutable<ILogicalExpression>> outVarRefList) {
for (LogicalVariable var : varList) {
VariableReferenceExpression varRef = new VariableReferenceExpression(var);
varRef.setSourceLocation(sourceLoc);
- varRefs.add(new MutableObject<>(varRef));
+ outVarRefList.add(new MutableObject<>(varRef));
}
- return varRefs;
}
}
[asterixdb] 21/25: [NO ISSUE][MISC] Advance it.unimi.dsi:fastutil
8.3.0->8.5.4
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d821af07316d94911f153acfae7c641d8216e820
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Apr 5 09:33:15 2021 -0400
[NO ISSUE][MISC] Advance it.unimi.dsi:fastutil 8.3.0->8.5.4
Change-Id: Idd434fcf59eeb81638ebbe96d99fb61c2965da90
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10903
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
asterixdb/pom.xml | 2 +-
.../appended-resources/supplemental-models.xml | 32 ++++++++++++++++++----
.../algebricks/algebricks-core/pom.xml | 2 +-
.../hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +-
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 2 +-
hyracks-fullstack/pom.xml | 4 +--
6 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index f89f0ac..6067ef9 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1390,7 +1390,7 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
- <version>8.3.0</version>
+ <version>8.5.4</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index bcdda15..7f3b7ec 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -250,11 +250,33 @@
<artifactId>fastutil</artifactId>
<properties>
<!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <!-- see https://github.com/vigna/fastutil/blob/8.1.1/LICENSE-2.0 -->
- <!-- see https://github.com/vigna/fastutil/blob/8.2.2/LICENSE-2.0 -->
- <!-- see https://github.com/vigna/fastutil/blob/8.3.0/LICENSE-2.0 -->
- <license.ignoreMissingEmbeddedLicense>8.1.1,8.2.2,8.3.0</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>8.1.1,8.2.2,8.3.0</license.ignoreMissingEmbeddedNotice>
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-core</artifactId>
+ <properties>
+ <!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
+ </properties>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil-extra</artifactId>
+ <properties>
+ <!-- fastutil is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
+ <!-- see https://github.com/vigna/fastutil/blob/8.5.4/LICENSE-2.0 -->
+ <license.ignoreMissingEmbeddedLicense>8.5.4</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>8.5.4</license.ignoreMissingEmbeddedNotice>
</properties>
</project>
</supplement>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
index b6c19f0..f58a7f6 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
@@ -104,7 +104,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
index a11d8de..fb143d9 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml
@@ -54,7 +54,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/pom.xml b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
index 4993443..a03e884 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
@@ -101,7 +101,7 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
+ <artifactId>fastutil-core</artifactId>
</dependency>
</dependencies>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 0fb88ed..27f9ab1 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -307,8 +307,8 @@
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- <version>8.3.0</version>
+ <artifactId>fastutil-core</artifactId>
+ <version>8.5.4</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
[asterixdb] 22/25: [NO ISSUE][HYR][UTIL] Abort retries on expired
span no retry delay
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 244c3c5e8c12f5ab7699609fdc47ffad6d2675b0
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Apr 6 12:39:30 2021 -0400
[NO ISSUE][HYR][UTIL] Abort retries on expired span no retry delay
Change-Id: I2f56cc23c5a869e76241f1ac303cf57a14714182
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10923
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Tested-by: Michael Blow <mb...@apache.org>
---
.../src/main/java/org/apache/hyracks/api/util/InvokeUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index 6ecb677..0b1c5a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@ -278,7 +278,7 @@ public class InvokeUtil {
failure = th;
try {
long delayMs = delay.calculate(attempt);
- if (!policy.retry(th) || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
+ if (!policy.retry(th) || span.elapsed() || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
onFailure.attemptFailed(action, attempt, true, span, failure);
throw HyracksDataException.create(failure);
} else {
[asterixdb] 02/25: [NO ISSUE][OTH] Update storage options names
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 690da336312dacb2a42f4c21e71a8ad1bb7417f5
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Fri Mar 26 18:32:10 2021 +0300
[NO ISSUE][OTH] Update storage options names
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Update storage options names and descriptions to better
reflect their usage.
Change-Id: I436c6327fb6b6af432e5cf74962beca38660b9c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10723
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +++++++--------
.../asterix/common/config/StorageProperties.java | 31 +++++++++++-----------
2 files changed, 26 insertions(+), 27 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index a9a3a3e..9aa433f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -209,15 +209,15 @@ public class NCAppRuntimeContext implements INcApplicationContext {
}
localResourceRepository.deleteStorageData();
}
- int maxConcurrentFlushes = storageProperties.getMaxConcurrentFlushes();
- if (maxConcurrentFlushes <= 0) {
- maxConcurrentFlushes = ioManager.getIODevices().size();
+ int maxScheduledFlushes = storageProperties.getMaxScheduledFlushes();
+ if (maxScheduledFlushes <= 0) {
+ maxScheduledFlushes = ioManager.getIODevices().size();
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("The value of maxConcurrentFlushes is not provided. Setting maxConcurrentFlushes = {}.",
- maxConcurrentFlushes);
+ LOGGER.info("The value of maxScheduledFlushes is not provided. Setting maxConcurrentFlushes = {}.",
+ maxScheduledFlushes);
}
}
- virtualBufferCache = new GlobalVirtualBufferCache(allocator, storageProperties, maxConcurrentFlushes);
+ virtualBufferCache = new GlobalVirtualBufferCache(allocator, storageProperties, maxScheduledFlushes);
// Must start vbc now instead of by life cycle component manager (lccm) because lccm happens after
// the metadata bootstrap task
((ILifeCycleComponent) virtualBufferCache).start();
@@ -584,24 +584,24 @@ public class NCAppRuntimeContext implements INcApplicationContext {
String schedulerName = storageProperties.getIoScheduler();
int numPartitions = ioManager.getIODevices().size();
- int maxRunningFlushes = storageProperties.getMaxRunningFlushes(numPartitions);
+ int maxConcurrentFlushes = storageProperties.geMaxConcurrentFlushes(numPartitions);
int maxScheduledMerges = storageProperties.getMaxScheduledMerges(numPartitions);
- int maxRunningMerges = storageProperties.getMaxRunningMerges(numPartitions);
+ int maxConcurrentMerges = storageProperties.getMaxConcurrentMerges(numPartitions);
ILSMIOOperationScheduler ioScheduler = null;
if (AsynchronousScheduler.FACTORY.getName().equalsIgnoreCase(schedulerName)) {
ioScheduler = AsynchronousScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
} else if (GreedyScheduler.FACTORY.getName().equalsIgnoreCase(schedulerName)) {
ioScheduler = GreedyScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
} else {
if (LOGGER.isWarnEnabled()) {
LOGGER.log(Level.WARN,
"Unknown storage I/O scheduler: " + schedulerName + "; defaulting to greedy I/O scheduler.");
}
ioScheduler = GreedyScheduler.FACTORY.createIoScheduler(getServiceContext().getThreadFactory(),
- HaltCallback.INSTANCE, maxRunningFlushes, maxScheduledMerges, maxRunningMerges);
+ HaltCallback.INSTANCE, maxConcurrentFlushes, maxScheduledMerges, maxConcurrentMerges);
}
return ioScheduler;
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index fc33b1a..d9463bf 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -19,7 +19,6 @@
package org.apache.asterix.common.config;
import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE;
-import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
@@ -49,16 +48,16 @@ public class StorageProperties extends AbstractProperties {
STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(POSITIVE_INTEGER, 2),
STORAGE_MEMORYCOMPONENT_FLUSH_THRESHOLD(DOUBLE, 0.9d),
- STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES(INTEGER, 0),
+ STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES(NONNEGATIVE_INTEGER, 0),
STORAGE_FILTERED_MEMORYCOMPONENT_MAX_SIZE(LONG_BYTE_UNIT, 0L),
STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
STORAGE_COMPRESSION_BLOCK(STRING, "snappy"),
STORAGE_DISK_FORCE_BYTES(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(16, MEGABYTE)),
STORAGE_IO_SCHEDULER(STRING, "greedy"),
STORAGE_WRITE_RATE_LIMIT(LONG_BYTE_UNIT, 0l),
- STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
+ STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 8),
- STORAGE_MAX_RUNNING_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2);
+ STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -96,8 +95,8 @@ public class StorageProperties extends AbstractProperties {
return "The page size in bytes for pages allocated to memory components";
case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS:
return "The number of memory components to be used per lsm index";
- case STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES:
- return "The maximum number of concurrent flush operations. 0 means that the value will be "
+ case STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES:
+ return "The maximum number of scheduled flush operations. 0 means that the value will be "
+ "calculated as the number of partitions";
case STORAGE_MEMORYCOMPONENT_FLUSH_THRESHOLD:
return "The memory usage threshold when memory components should be flushed";
@@ -114,12 +113,12 @@ public class StorageProperties extends AbstractProperties {
return "The number of bytes before each disk force (fsync)";
case STORAGE_IO_SCHEDULER:
return "The I/O scheduler for LSM flush and merge operations";
- case STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION:
- return "The maximum number of running flushes per partition (0 means unlimited)";
+ case STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION:
+ return "The maximum number of concurrently executed flushes per partition (0 means unlimited)";
case STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION:
return "The maximum number of scheduled merges per partition (0 means unlimited)";
- case STORAGE_MAX_RUNNING_MERGES_PER_PARTITION:
- return "The maximum number of running merges per partition (0 means unlimited)";
+ case STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION:
+ return "The maximum number of concurrently executed merges per partition (0 means unlimited)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -189,8 +188,8 @@ public class StorageProperties extends AbstractProperties {
return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
}
- public int getMaxConcurrentFlushes() {
- return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_MAX_CONCURRENT_FLUSHES);
+ public int getMaxScheduledFlushes() {
+ return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_MAX_SCHEDULED_FLUSHES);
}
public long getJobExecutionMemoryBudget() {
@@ -213,8 +212,8 @@ public class StorageProperties extends AbstractProperties {
return accessor.getString(Option.STORAGE_IO_SCHEDULER);
}
- public int getMaxRunningFlushes(int numPartitions) {
- int value = accessor.getInt(Option.STORAGE_MAX_RUNNING_FLUSHES_PER_PARTITION);
+ public int geMaxConcurrentFlushes(int numPartitions) {
+ int value = accessor.getInt(Option.STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION);
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
@@ -223,8 +222,8 @@ public class StorageProperties extends AbstractProperties {
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
- public int getMaxRunningMerges(int numPartitions) {
- int value = accessor.getInt(Option.STORAGE_MAX_RUNNING_MERGES_PER_PARTITION);
+ public int getMaxConcurrentMerges(int numPartitions) {
+ int value = accessor.getInt(Option.STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION);
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
[asterixdb] 05/25: [ASTERIXDB-2854] Make UDF deploy timeout
configurable
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a016c1102c9f62031cbf667c8e43605925347402
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Mon Mar 22 13:04:28 2021 -0700
[ASTERIXDB-2854] Make UDF deploy timeout configurable
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add a config variable for UDF deployment timeout.
For very large libraries, this is necessary as
it can take some time for them to be distributed
and extracted on every node in the cluster
Change-Id: Ic318a3b70d5d44c1dbd26f543eb6ef5fe62002ac
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10623
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Ian Maxon <im...@uci.edu>
---
.../java/org/apache/asterix/api/http/server/NCUdfApiServlet.java | 4 +++-
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../java/org/apache/asterix/common/config/ExternalProperties.java | 8 +++++++-
5 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index f164938..717ebf8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -83,6 +83,7 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
protected Path workingDir;
protected String sysAuthHeader;
private ILibraryManager libraryManager;
+ private int timeout;
private static final Logger LOGGER = LogManager.getLogger();
@@ -91,6 +92,7 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort);
this.compilationProvider = compilationProvider;
this.receptionist = appCtx.getReceptionist();
+ this.timeout = appCtx.getExternalProperties().getLibraryDeployTimeout();
}
@Override
@@ -152,7 +154,7 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
InternalRequestResponse responseMsg;
try {
ncMb.sendMessageToPrimaryCC(requestMessage);
- responseMsg = (InternalRequestResponse) responseFuture.get(120000, TimeUnit.MILLISECONDS);
+ responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
} finally {
ncMb.deregisterMessageFuture(responseFuture.getFutureId());
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index b6a49a6..eba50dc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -24,6 +24,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "INFO",
"max\.wait\.active\.cluster" : 60,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index dedb40f..98faa65 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -24,6 +24,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 985d3bd..a92f7d1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -24,6 +24,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 9bef65d..46258bf 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -50,7 +50,8 @@ public class ExternalProperties extends AbstractProperties {
POSITIVE_INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(200, StorageUtil.StorageUnit.MEGABYTE),
"The maximum accepted web request size in bytes"),
- REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 50, "The maximum number of archived requests to maintain");
+ REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 50, "The maximum number of archived requests to maintain"),
+ LIBRARY_DEPLOY_TIMEOUT(POSITIVE_INTEGER, 1800, "Timeout to upload a UDF in seconds");
private final IOptionType type;
private final Object defaultValue;
@@ -76,6 +77,7 @@ public class ExternalProperties extends AbstractProperties {
case LOG_LEVEL:
case MAX_WAIT_ACTIVE_CLUSTER:
case MAX_WEB_REQUEST_SIZE:
+ case LIBRARY_DEPLOY_TIMEOUT:
return Section.COMMON;
case CC_JAVA_OPTS:
case NC_JAVA_OPTS:
@@ -149,4 +151,8 @@ public class ExternalProperties extends AbstractProperties {
return accessor.getInt(Option.REQUESTS_ARCHIVE_SIZE);
}
+ public int getLibraryDeployTimeout() {
+ return accessor.getInt(Option.LIBRARY_DEPLOY_TIMEOUT);
+ }
+
}
[asterixdb] 07/25: [ASTERIXDB-2855] Allow additions to Python UDF
env
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1dea45828390e3116ba317bd5e39c7d4b055ae62
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Mon Mar 22 16:50:49 2021 -0700
[ASTERIXDB-2855] Allow additions to Python UDF env
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add a new configuration parameter to allow
appending new environment variables to the
Python interpreter process used in Python
UDFs.
- Add test to check that it works.
- Skip adding empty args, path or env during
setup of Python UDF commandline
Change-Id: Ib6e1ee7debc9c2e07d24163542b1f98886792161
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10644
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../src/test/resources/TweetSent/roundtrip.py | 4 +++
asterixdb/asterix-app/src/test/resources/cc.conf | 1 +
.../py_function_error.2.ddl.sqlpp | 5 +++-
...2.ddl.sqlpp => py_function_error.5.query.sqlpp} | 12 ++++----
...2.ddl.sqlpp => py_function_error.6.query.sqlpp} | 12 ++++----
.../py_function_error/py_function_error.3.json | 1 +
.../py_function_error/py_function_error.4.json | 1 +
.../resources/runtimets/testsuite_it_python.xml | 2 +-
.../external/library/PythonLibraryEvaluator.java | 17 +++++++----
.../library/PythonLibraryEvaluatorFactory.java | 34 +++++++++++++++++++---
.../control/common/controllers/NCConfig.java | 4 +++
11 files changed, 71 insertions(+), 22 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
index 8b8fced..9058a01 100644
--- a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
+++ b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import math
+import os
def sqrt(num):
return math.sqrt(num)
@@ -26,3 +27,6 @@ class Tests(object):
def warning(self):
raise ArithmeticError("oof")
+
+ def env_test(self, key):
+ return os.environ[key]
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index e2cd5b9..9532849 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -34,6 +34,7 @@ nc.api.port=19005
[nc]
credential.file=src/test/resources/security/passwd
python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
address=127.0.0.1
command=asterixnc
app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
index 0ad9fb3..74fe03f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
@@ -23,4 +23,7 @@ create function warning()
as "roundtrip", "Tests.warning" at testlib;
create function roundtrip(s)
- as "roundtrip", "Tests.roundtrip" at testlib;
\ No newline at end of file
+ as "roundtrip", "Tests.roundtrip" at testlib;
+
+create function env_test(k)
+ as "roundtrip", "Tests.env_test" at testlib;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
index 0ad9fb3..7334470 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.5.query.sqlpp
@@ -16,11 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+/*
+* Description : Access a records nested records at each level.
+* Expected Res : Success
+* Date : 04 Jun 2015
+*/
+// param max-warnings:json=0
use test;
-create function warning()
- as "roundtrip", "Tests.warning" at testlib;
-
-create function roundtrip(s)
- as "roundtrip", "Tests.roundtrip" at testlib;
\ No newline at end of file
+env_test("FOO");
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
index 0ad9fb3..092af06 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_function_error/py_function_error.6.query.sqlpp
@@ -16,11 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+/*
+* Description : Access a records nested records at each level.
+* Expected Res : Success
+* Date : 04 Jun 2015
+*/
+// param max-warnings:json=0
use test;
-create function warning()
- as "roundtrip", "Tests.warning" at testlib;
-
-create function roundtrip(s)
- as "roundtrip", "Tests.roundtrip" at testlib;
\ No newline at end of file
+env_test("BAR");
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json
new file mode 100644
index 0000000..58de662
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.3.json
@@ -0,0 +1 @@
+"BAR=BAZ"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json
new file mode 100644
index 0000000..f73df98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_function_error/py_function_error.4.json
@@ -0,0 +1 @@
+"BAZ"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index 4e1d5b2..35bec85 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -57,7 +57,7 @@
result[0].append(self.next_tuple(*arg, key=self.mid))
File "entrypoint.py", line 99, in next_tuple
return self.wrapped_fns[key](*args)
- File "site-packages/roundtrip.py", line 28, in warning
+ File "site-packages/roundtrip.py", line 29, in warning
raise ArithmeticError("oof")
ArithmeticError: oof
(in line 28, at column 1)</expected-warn>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
index e2229ee..457b86a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluator.java
@@ -27,6 +27,7 @@ import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -65,18 +66,21 @@ public class PythonLibraryEvaluator extends AbstractStateObject implements IDeal
private IPCSystem ipcSys;
private String sitePkgs;
private List<String> pythonArgs;
+ private Map<String, String> pythonEnv;
private TaskAttemptId task;
private IWarningCollector warningCollector;
private SourceLocation sourceLoc;
public PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, ILibraryManager libMgr,
- File pythonHome, String sitePkgs, List<String> pythonArgs, ExternalFunctionResultRouter router,
- IPCSystem ipcSys, TaskAttemptId task, IWarningCollector warningCollector, SourceLocation sourceLoc) {
+ File pythonHome, String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv,
+ ExternalFunctionResultRouter router, IPCSystem ipcSys, TaskAttemptId task,
+ IWarningCollector warningCollector, SourceLocation sourceLoc) {
super(jobId, evaluatorId);
this.libMgr = libMgr;
this.pythonHome = pythonHome;
this.sitePkgs = sitePkgs;
this.pythonArgs = pythonArgs;
+ this.pythonEnv = pythonEnv;
this.router = router;
this.task = task;
this.ipcSys = ipcSys;
@@ -98,8 +102,8 @@ public class PythonLibraryEvaluator extends AbstractStateObject implements IDeal
args.add(InetAddress.getLoopbackAddress().getHostAddress());
args.add(Integer.toString(port));
args.add(sitePkgs);
-
ProcessBuilder pb = new ProcessBuilder(args.toArray(new String[0]));
+ pb.environment().putAll(pythonEnv);
pb.directory(new File(wd));
p = pb.start();
proto = new PythonIPCProto(p.getOutputStream(), router, p);
@@ -199,14 +203,15 @@ public class PythonLibraryEvaluator extends AbstractStateObject implements IDeal
public static PythonLibraryEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
ExternalFunctionResultRouter router, IPCSystem ipcSys, File pythonHome, IHyracksTaskContext ctx,
- String sitePkgs, List<String> pythonArgs, IWarningCollector warningCollector, SourceLocation sourceLoc)
- throws IOException, AsterixException {
+ String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv, IWarningCollector warningCollector,
+ SourceLocation sourceLoc) throws IOException, AsterixException {
PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryEvaluator evaluator = (PythonLibraryEvaluator) ctx.getStateObject(evaluatorId);
if (evaluator == null) {
evaluator = new PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, libMgr, pythonHome,
- sitePkgs, pythonArgs, router, ipcSys, ctx.getTaskAttemptId(), warningCollector, sourceLoc);
+ sitePkgs, pythonArgs, pythonEnv, router, ipcSys, ctx.getTaskAttemptId(), warningCollector,
+ sourceLoc);
ctx.getJobletContext().registerDeallocatable(evaluator);
evaluator.initialize();
ctx.setStateObject(evaluator);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
index 86d51de..06c9bc9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorFactory.java
@@ -23,8 +23,9 @@ import static org.apache.asterix.external.library.PythonLibraryEvaluator.SITE_PA
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -46,6 +47,7 @@ public class PythonLibraryEvaluatorFactory {
private final ExternalFunctionResultRouter router;
private final String sitePackagesPath;
private final List<String> pythonArgs;
+ private final Map<String, String> pythonEnv;
public PythonLibraryEvaluatorFactory(IHyracksTaskContext ctx) throws AsterixException {
this.ctx = ctx;
@@ -67,17 +69,41 @@ public class PythonLibraryEvaluatorFactory {
+ NCConfig.Option.PYTHON_CMD_AUTOLOCATE.ini() + " is false");
}
}
+ pythonEnv = new HashMap<>();
+ String[] envRaw = appCfg.getStringArray((NCConfig.Option.PYTHON_ENV));
+ if (envRaw != null) {
+ for (String rawEnvArg : envRaw) {
+ //TODO: i think equals is shared among all unixes and windows. but it needs verification
+ if (rawEnvArg.length() < 1) {
+ continue;
+ }
+ String[] rawArgSplit = rawEnvArg.split("(?<!\\\\)=", 2);
+ if (rawArgSplit.length < 2) {
+ throw AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION,
+ "Invalid environment variable format detected.");
+ }
+ pythonEnv.put(rawArgSplit[0], rawArgSplit[1]);
+ }
+ }
pythonPath = new File(pythonPathCmd);
List<String> sitePkgs = new ArrayList<>();
sitePkgs.add(SITE_PACKAGES);
String[] addlSitePackages = appCfg.getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
- sitePkgs.addAll(Arrays.asList(addlSitePackages));
+ for (String sitePkg : addlSitePackages) {
+ if (sitePkg.length() > 0) {
+ sitePkgs.add(sitePkg);
+ }
+ }
if (appCfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) {
sitePkgs.add("ipc" + File.separator + SITE_PACKAGES + File.separator);
}
String[] pythonArgsRaw = appCfg.getStringArray(NCConfig.Option.PYTHON_ARGS);
if (pythonArgsRaw != null) {
- pythonArgs.addAll(Arrays.asList(pythonArgsRaw));
+ for (String arg : pythonArgsRaw) {
+ if (arg.length() > 0) {
+ pythonArgs.add(arg);
+ }
+ }
}
StringBuilder sitePackagesPathBuilder = new StringBuilder();
for (int i = 0; i < sitePkgs.size() - 1; i++) {
@@ -91,6 +117,6 @@ public class PythonLibraryEvaluatorFactory {
public PythonLibraryEvaluator getEvaluator(IExternalFunctionInfo fnInfo, SourceLocation sourceLoc)
throws IOException, AsterixException {
return PythonLibraryEvaluator.getInstance(fnInfo, libraryManager, router, ipcSys, pythonPath, ctx,
- sitePackagesPath, pythonArgs, ctx.getWarningCollector(), sourceLoc);
+ sitePackagesPath, pythonArgs, pythonEnv, ctx.getWarningCollector(), sourceLoc);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 22b240a..01cb9bf 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -98,6 +98,7 @@ public class NCConfig extends ControllerConfig {
PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, new String[0]),
PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
PYTHON_ARGS(STRING_ARRAY, (String[]) null),
+ PYTHON_ENV(STRING_ARRAY, (String[]) null),
CREDENTIAL_FILE(
OptionTypes.STRING,
(Function<IApplicationConfig, String>) appConfig -> FileUtil
@@ -245,6 +246,8 @@ public class NCConfig extends ControllerConfig {
return "Python args to pass to Python interpreter";
case PYTHON_CMD_AUTOLOCATE:
return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter";
+ case PYTHON_ENV:
+ return "List of environment variables to set when invoking the Python interpreter for Python UDFs. E.g. FOO=1";
case CREDENTIAL_FILE:
return "Path to HTTP basic credentials";
default:
@@ -621,4 +624,5 @@ public class NCConfig extends ControllerConfig {
public String getCredentialFilePath() {
return getAppConfig().getString(Option.CREDENTIAL_FILE);
}
+
}
[asterixdb] 15/25: [NO ISSUE][OTH] Add an identifier mapper to map
identifiers
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7b2191a9c524eac57b066b8fee3e9ff149c2eca0
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Wed Mar 31 23:30:47 2021 -0700
[NO ISSUE][OTH] Add an identifier mapper to map identifiers
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Add an identifier mapper to map identifiers such as "dataset" to allow
extensions to provide their mapping for the identifier.
Error messages in the code should use the identifier.
- Add source location and parameters to IFormattedException
- Add methods to get the source location and parameters to IFormattedException
- Include the parameters of the warning message in the warning object
Change-Id: I4bd0a602fddeb09a7dfca64ce35f5eee6f3c4777
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9884
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../operators/physical/InvertedIndexPOperator.java | 4 +-
.../rules/MetaFunctionToMetaVariableRule.java | 4 +-
.../rules/SetAsterixPhysicalOperatorsRule.java | 5 ++-
.../asterix/translator/AbstractLangTranslator.java | 21 +++++++----
.../translator/LangExpressionToPlanTranslator.java | 22 ++++++-----
.../api/http/server/ConnectorApiServlet.java | 7 +++-
.../api/http/server/RebalanceApiServlet.java | 5 ++-
.../asterix/app/function/DatasetRewriter.java | 6 ++-
.../apache/asterix/app/function/FeedRewriter.java | 5 ++-
.../asterix/app/translator/QueryTranslator.java | 43 ++++++++++++----------
.../test/resources/runtimets/testsuite_sqlpp.xml | 26 ++++++-------
.../asterix/common/api/IIdentifierMapper.java | 27 ++++++++++++++
.../asterix/common/config/CompilerProperties.java | 2 +-
.../common/config/TransactionProperties.java | 5 ++-
.../asterix/common/exceptions/WarningUtil.java | 4 +-
.../common/utils/IdentifierMappingUtil.java | 41 +++++++++++++++++++++
.../asterix/common/utils/IdentifierUtil.java | 34 +++++++++++++++++
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../asterix-doc/src/main/markdown/sqlpp/5_error.md | 2 +-
.../org/apache/asterix/metadata/MetadataNode.java | 15 ++++----
.../metadata/declared/MetadataManagerUtil.java | 8 +++-
.../metadata/declared/MetadataProvider.java | 18 +++++----
.../apache/asterix/metadata/entities/Function.java | 7 ++++
.../asterix/metadata/feeds/FeedMetadataUtil.java | 9 +++--
.../apache/asterix/metadata/utils/DatasetUtil.java | 6 ++-
.../InvertedIndexResourceFactoryProvider.java | 5 ++-
.../common/exceptions/AlgebricksException.java | 2 +
.../hyracks/api/exceptions/HyracksException.java | 2 +
.../api/exceptions/IFormattedException.java | 11 ++++++
.../org/apache/hyracks/api/exceptions/Warning.java | 27 ++++++++++++--
30 files changed, 285 insertions(+), 92 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 491911b..4ad7d3f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.algebra.operators.physical;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -155,7 +157,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
dataset.getDataverseName(), dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + datasetName);
+ "Code generation error: no index " + indexName + " for " + dataset() + " " + datasetName);
}
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap);
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
index b6dbbe7..aa2b8fd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -254,7 +256,7 @@ class LogicalExpressionReferenceTransform implements ILogicalExpressionReference
// the user query provides zero parameter for the meta function.
if (variableRequired) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, expr.getSourceLocation(),
- "Cannot resolve ambiguous meta function call. There are more than one dataset choice!");
+ "Cannot resolve ambiguous meta function call. There are more than one " + dataset() + " choice");
}
VariableReferenceExpression metaVarRef = new VariableReferenceExpression(metaVar);
metaVarRef.setSourceLocation(expr.getSourceLocation());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
index d466446..5ec1a7f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -265,7 +267,8 @@ public final class SetAsterixPhysicalOperatorsRule extends SetAlgebricksPhysical
INodeDomain storageDomain = mp.findNodeDomain(dataset.getNodeGroupName());
if (dsi == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, op.getSourceLocation(),
- "Could not find index " + jobGenParams.getIndexName() + " for dataset " + dataSourceId);
+ "Could not find index " + jobGenParams.getIndexName() + " for " + dataset() + " "
+ + dataSourceId);
}
IndexType indexType = jobGenParams.getIndexType();
boolean requiresBroadcast = jobGenParams.getRequiresBroadcast();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 58bdaca..6f07718 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
@@ -57,6 +60,7 @@ import org.apache.asterix.lang.common.statement.UpsertStatement;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -73,11 +77,12 @@ public abstract class AbstractLangTranslator {
protected static final String INVALID_OPERATION_MESSAGE = "Invalid operation - %s";
- protected static final String BAD_DATAVERSE_DML_MESSAGE = "%s operation is not permitted in dataverse %s";
+ protected static final String BAD_DATAVERSE_DML_MESSAGE = "%s operation is not permitted in " + dataverse() + " %s";
- protected static final String BAD_DATAVERSE_DDL_MESSAGE = "Cannot %s dataverse: %s";
+ protected static final String BAD_DATAVERSE_DDL_MESSAGE = "Cannot %s " + dataverse() + ": %s";
- protected static final String BAD_DATAVERSE_OBJECT_DDL_MESSAGE = "Cannot %s a %s belonging to the dataverse: %s";
+ protected static final String BAD_DATAVERSE_OBJECT_DDL_MESSAGE =
+ "Cannot %s a %s belonging to the " + dataverse() + ": %s";
public void validateOperation(ICcApplicationContext appCtx, Dataverse defaultDataverse, Statement stmt)
throws AlgebricksException {
@@ -210,7 +215,7 @@ public abstract class AbstractLangTranslator {
}
invalidOperation = isMetadataDataverse(dataverseName);
if (invalidOperation) {
- message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "create", "dataset", dataverseName);
+ message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "create", dataset(), dataverseName);
}
if (!invalidOperation) {
@@ -221,9 +226,9 @@ public abstract class AbstractLangTranslator {
Pair<Boolean, String> validationResult =
DatasetHints.validate(appCtx, hint.getKey(), hint.getValue());
if (!validationResult.first) {
- errorMsgBuffer.append("Dataset: ").append(dsCreateStmt.getName().getValue())
- .append(" error in processing hint: ").append(hint.getKey()).append(" ")
- .append(validationResult.second);
+ errorMsgBuffer.append(StringUtils.capitalize(dataset())).append(": ")
+ .append(dsCreateStmt.getName().getValue()).append(" error in processing hint: ")
+ .append(hint.getKey()).append(" ").append(validationResult.second);
errorMsgBuffer.append(" \n");
}
}
@@ -242,7 +247,7 @@ public abstract class AbstractLangTranslator {
}
invalidOperation = isMetadataDataverse(dataverseName);
if (invalidOperation) {
- message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "drop", "dataset", dataverseName);
+ message = String.format(BAD_DATAVERSE_OBJECT_DDL_MESSAGE, "drop", dataset(), dataverseName);
}
break;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index fd689a5..da07acb 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -206,8 +208,8 @@ abstract class LangExpressionToPlanTranslator
validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName(), sourceLoc);
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
if (dataset.hasMetaPart()) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, dataset.getDatasetName() + ": load "
+ + dataset() + " is not supported on " + dataset() + "s with meta records");
}
LoadableDataSource lds;
@@ -430,8 +432,8 @@ abstract class LangExpressionToPlanTranslator
SourceLocation sourceLoc = stmt.getSourceLocation();
if (targetDatasource.getDataset().hasMetaPart()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": delete from dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": delete from " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
List<String> filterField = DatasetUtil.getFilterField(targetDatasource.getDataset());
@@ -461,8 +463,8 @@ abstract class LangExpressionToPlanTranslator
SourceLocation sourceLoc = stmt.getSourceLocation();
if (!targetDatasource.getDataset().allow(topOp, DatasetUtil.OP_UPSERT)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": upsert into dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": upsert into " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
ProjectOperator project = (ProjectOperator) topOp;
CompiledUpsertStatement compiledUpsert = (CompiledUpsertStatement) stmt;
@@ -474,7 +476,7 @@ abstract class LangExpressionToPlanTranslator
if (targetDatasource.getDataset().hasMetaPart()) {
if (returnExpression != null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Returning not allowed on datasets with Meta records");
+ "Returning not allowed on " + dataset() + "s with meta records");
}
List<LogicalVariable> metaAndKeysVars;
List<Mutable<ILogicalExpression>> metaAndKeysExprs;
@@ -585,8 +587,8 @@ abstract class LangExpressionToPlanTranslator
SourceLocation sourceLoc = stmt.getSourceLocation();
if (targetDatasource.getDataset().hasMetaPart()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- targetDatasource.getDataset().getDatasetName()
- + ": insert into dataset is not supported on Datasets with Meta records");
+ targetDatasource.getDataset().getDatasetName() + ": insert into " + dataset()
+ + " is not supported on " + dataset() + "s with meta records");
}
List<String> filterField = DatasetUtil.getFilterField(targetDatasource.getDataset());
@@ -683,7 +685,7 @@ abstract class LangExpressionToPlanTranslator
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Cannot write output to an external dataset.");
+ "Cannot write output to an external " + dataset());
}
DataSourceId sourceId = new DataSourceId(dataverseName, datasetName);
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
index 2fafcfc..353b849 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
@@ -19,6 +19,8 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import java.io.IOException;
import java.io.PrintWriter;
@@ -34,6 +36,7 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.utils.FlushDatasetUtil;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.io.FileSplit;
@@ -98,8 +101,8 @@ public class ConnectorApiServlet extends AbstractServlet {
metadataProvider.setMetadataTxnContext(mdTxnCtx);
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- jsonResponse.put("error",
- "Dataset " + datasetName + " does not exist in " + "dataverse " + dataverseName);
+ jsonResponse.put("error", StringUtils.capitalize(dataset()) + " " + datasetName
+ + " does not exist in " + dataverse() + " " + dataverseName);
out.write(jsonResponse.toString());
out.flush();
return;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
index 8b04cc3..c4d664b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
@@ -19,6 +19,7 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
@@ -121,13 +122,13 @@ public class RebalanceApiServlet extends AbstractServlet {
// If a user gives parameter datasetName, she should give dataverseName as well.
if (dataverseName == null && datasetName != null) {
sendResponse(response, HttpResponseStatus.BAD_REQUEST,
- "to rebalance a particular dataset, the parameter dataverseName must be given");
+ "to rebalance a particular " + dataset() + ", the parameter dataverseName must be given");
return;
}
// Does not allow rebalancing a metadata dataset.
if (MetadataConstants.METADATA_DATAVERSE_NAME.equals(dataverseName)) {
- sendResponse(response, HttpResponseStatus.BAD_REQUEST, "cannot rebalance a metadata dataset");
+ sendResponse(response, HttpResponseStatus.BAD_REQUEST, "cannot rebalance a metadata " + dataset());
return;
}
// Schedules a rebalance task and wait for its completion.
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
index 974cd9e..0d20d51 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -67,7 +69,7 @@ public class DatasetRewriter implements IFunctionToDataSourceRewriter, IResultTy
if (unnest.getPositionalVariable() != null) {
// TODO remove this after enabling the support of positional variables in data scan
throw new CompilationException(ErrorCode.COMPILATION_ERROR, unnest.getSourceLocation(),
- "No positional variables are allowed over datasets.");
+ "No positional variables are allowed over " + dataset() + "s");
}
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
@@ -122,7 +124,7 @@ public class DatasetRewriter implements IFunctionToDataSourceRewriter, IResultTy
IAType type = metadata.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
if (type == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, datasetFnCall.getSourceLocation(),
- "No type for dataset " + dataset.getDatasetName());
+ "No type for " + dataset() + " " + dataset.getDatasetName());
}
return type;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
index 90374c3..b01ea65 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FeedRewriter.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.List;
@@ -129,7 +131,8 @@ public class FeedRewriter implements IFunctionToDataSourceRewriter, IResultTypeC
if (dataset.hasMetaPart()) {
String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getConfiguration());
if (metaTypeName == null) {
- throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
+ throw new AlgebricksException(
+ "Feed to a " + dataset() + " with metadata doesn't have meta type specified");
}
metaType = (ARecordType) metadataProvider.findType(id.getDataverseName(), metaTypeName);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ab682c1..d8698f9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.app.translator;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
@@ -576,7 +579,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
(ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance();
if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "The correlated-prefix merge policy cannot be used with external dataset.");
+ "The correlated-prefix merge policy cannot be used with external " + dataset() + "s");
}
if (compactionPolicyProperties == null) {
if (mergePolicyFactory.getName().compareTo("no-merge") != 0) {
@@ -813,8 +816,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
TransactionState.COMMIT);
break;
default:
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Unknown dataset type " + dsType);
+ throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE,
+ dataset.getDatasetType().toString());
}
// #. initialize DatasetIdFactory if it is not initialized.
@@ -921,7 +924,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
SourceLocation sourceLoc) throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- String.format("Dataset %s has to be a record type.", isMetaItemType ? "meta type" : "type"));
+ String.format(StringUtils.capitalize(dataset()) + " %s has to be a record type.",
+ isMetaItemType ? "meta type" : "type"));
}
}
@@ -1162,13 +1166,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
// Check if the dataset is indexible
if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "dataset using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
- + " Adapter can't be indexed");
+ dataset() + " using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
+ + " adapter can't be indexed");
}
// Check if the name of the index is valid
if (!ExternalIndexingOperations.isValidIndexName(index.getDatasetName(), index.getIndexName())) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "external dataset index name is invalid");
+ "external " + dataset() + " index name is invalid");
}
// Check if the files index exist
@@ -1209,7 +1213,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
metadataProvider);
if (spec == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Failed to create job spec for replicating Files Index For external dataset");
+ "Failed to create job spec for replicating files index for external " + dataset());
}
filesIndexReplicated = true;
runJob(hcc, spec, jobFlags);
@@ -1457,7 +1461,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)
|| dataverseName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- dataverseName + " dataverse can't be dropped");
+ dataverseName + " " + dataverse() + " can't be dropped");
}
lockUtil.dropDataverseBegin(lockManager, metadataProvider.getLocks(), dataverseName);
try {
@@ -1824,7 +1828,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
} else if (ExternalIndexingOperations.isFileIndex(index)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Dropping a dataset's files index is not allowed.");
+ "Dropping " + dataset() + " files index is not allowed.");
}
ensureNonPrimaryIndexDrop(index, sourceLoc);
// #. prepare a job to drop the index in NC.
@@ -3044,8 +3048,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
ActiveEntityEventsListener listener =
(ActiveEntityEventsListener) activeNotificationHandler.getListener(feedId);
if (listener != null && listener.getState() != ActivityState.STOPPED) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Feed " + feedId
- + " is currently active and connected to the following dataset(s) \n" + listener.toString());
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Feed " + feedId + " is currently active and connected to the following " + dataset() + "(s) \n"
+ + listener.toString());
} else if (listener != null) {
listener.unregister();
}
@@ -3071,7 +3076,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
if (feedPolicy == null) {
if (!stmtFeedPolicyDrop.getIfExists()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Unknown policy " + policyName + " in dataverse " + dataverseName);
+ "Unknown policy " + policyName + " in " + dataverse() + " " + dataverseName);
}
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
return;
@@ -3202,7 +3207,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
feedName, datasetName);
if (fc != null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Feed" + feedName + " is already connected dataset " + datasetName);
+ "Feed" + feedName + " is already connected to " + dataset() + " " + datasetName);
}
fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, whereClauseBody,
outputType.getTypeName());
@@ -3286,7 +3291,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (indexes.isEmpty()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Cannot compact the extrenal dataset " + datasetName + " because it has no indexes");
+ "Cannot compact the external " + dataset() + " " + datasetName + " because it has no indexes");
}
Dataverse dataverse =
MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName);
@@ -3599,14 +3604,14 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
// Dataset external ?
if (ds.getDatasetType() != DatasetType.EXTERNAL) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "dataset " + datasetName + " in dataverse " + dataverseName + " is not an external dataset");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, dataset() + " " + datasetName
+ + " in " + dataverse() + " " + dataverseName + " is not an external " + dataset());
}
// Dataset has indexes ?
indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (indexes.isEmpty()) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "External dataset " + datasetName
- + " in dataverse " + dataverseName + " doesn't have any index");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "External " + dataset() + " "
+ + datasetName + " in " + dataverse() + " " + dataverseName + " doesn't have any index");
}
// Record transaction time
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index ce440c1..19e0965 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -4855,7 +4855,7 @@
<test-case FilePath="dml">
<compilation-unit name="upsert-dataset-with-meta">
<output-dir compare="Text">upsert-dataset-with-meta</output-dir>
- <expected-error>upsert into dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>upsert into dataset is not supported on datasets with meta record</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
@@ -4881,13 +4881,13 @@
<test-case FilePath="dml">
<compilation-unit name="delete-dataset-with-meta">
<output-dir compare="Text">delete-dataset-with-meta</output-dir>
- <expected-error>delete from dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>delete from dataset is not supported on datasets with meta records</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
<compilation-unit name="insert-dataset-with-meta">
<output-dir compare="Text">insert-dataset-with-meta</output-dir>
- <expected-error>insert into dataset is not supported on Datasets with Meta records</expected-error>
+ <expected-error>insert into dataset is not supported on datasets with meta records</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="dml">
@@ -6692,7 +6692,7 @@
<output-dir compare="Text">partition-by-nonexistent-field</output-dir>
<expected-error>Field "id" is not found</expected-error>
<expected-error>Cannot find dataset with name testds in dataverse test</expected-error>
- <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds!</expected-error>
+ <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -11968,26 +11968,26 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-1">
<output-dir compare="Text">bad-function-ddl-1</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages!</expected-error>
- <expected-error>Cannot find dataset TweetMessages2 in dataverse experiments2 nor an alias with name TweetMessages2!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
+ <expected-error>Cannot find dataset TweetMessages2 in dataverse experiments2 nor an alias with name TweetMessages2</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-2">
<output-dir compare="Text">bad-function-ddl-2</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments2 nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments2 nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-3">
<output-dir compare="Text">bad-function-ddl-3</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experiments nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-4">
<output-dir compare="Text">bad-function-ddl-4</output-dir>
- <expected-error>Cannot find dataset TweetMessages in dataverse experients nor an alias with name TweetMessages!</expected-error>
+ <expected-error>Cannot find dataset TweetMessages in dataverse experients nor an alias with name TweetMessages</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
@@ -12006,13 +12006,13 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-7">
<output-dir compare="Text">bad-function-ddl-7</output-dir>
- <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes!</expected-error>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="bad-function-ddl-8">
<output-dir compare="Text">bad-function-ddl-8</output-dir>
- <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes!</expected-error>
+ <expected-error>Cannot find dataset TweetMessaes in dataverse experiments nor an alias with name TweetMessaes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
@@ -12500,7 +12500,7 @@
<test-case FilePath="load">
<compilation-unit name="dataset-with-meta">
<output-dir compare="Text">dataset-with-meta</output-dir>
- <expected-error>ASX1079: Compilation error: DatasetWithMeta: load dataset is not supported on Datasets with Meta records (in line 27, at column 1)</expected-error>
+ <expected-error>ASX1079: Compilation error: DatasetWithMeta: load dataset is not supported on datasets with meta records (in line 27, at column 1)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="load">
@@ -12868,7 +12868,7 @@
<test-case FilePath="meta">
<compilation-unit name="query_dataset_with_meta_failure">
<output-dir compare="Text">query_dataset_with_meta_failure</output-dir>
- <expected-error>ASX1079: Compilation error: Cannot resolve ambiguous meta function call. There are more than one dataset choice! (in line 24, at column 7)</expected-error>
+ <expected-error>ASX1079: Compilation error: Cannot resolve ambiguous meta function call. There are more than one dataset choice (in line 24, at column 7)</expected-error>
</compilation-unit>
</test-case>
</test-group>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java
new file mode 100644
index 0000000..8687239
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IIdentifierMapper.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.api;
+
+@FunctionalInterface
+public interface IIdentifierMapper {
+
+ String map(String identifier);
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 39142e5..866d183 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -83,7 +83,7 @@ public class CompilerProperties extends AbstractProperties {
COMPILER_EXTERNAL_FIELD_PUSHDOWN(
BOOLEAN,
AlgebricksConfig.EXTERNAL_FIELD_PUSHDOWN_DEFAULT,
- "Enable pushdown of field accesses to the external dataset data-scan operator"),
+ "Enable pushdown of field accesses to the external data-scan operator"),
COMPILER_SUBPLAN_MERGE(
BOOLEAN,
AlgebricksConfig.SUBPLAN_MERGE_DEFAULT,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
index 050a493..d67e9a6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
@@ -41,7 +42,7 @@ public class TransactionProperties extends AbstractProperties {
TXN_DATASET_CHECKPOINT_INTERVAL(
POSITIVE_INTEGER,
(int) TimeUnit.MINUTES.toSeconds(60),
- "The interval (in seconds) after which a dataset is considered idle and persisted to disk"),
+ "The interval (in seconds) after which a " + dataset() + " is considered idle and persisted to disk"),
TXN_LOG_BUFFER_NUMPAGES(POSITIVE_INTEGER, 8, "The number of pages in the transaction log tail"),
TXN_LOG_BUFFER_PAGESIZE(
INTEGER_BYTE_UNIT,
@@ -65,7 +66,7 @@ public class TransactionProperties extends AbstractProperties {
TXN_LOCK_ESCALATIONTHRESHOLD(
NONNEGATIVE_INTEGER,
1000,
- "The maximum number of entity locks to obtain before upgrading to a dataset lock"),
+ "The maximum number of entity locks to obtain before upgrading to a " + dataset() + " lock"),
TXN_LOCK_SHRINKTIMER(
POSITIVE_INTEGER,
5000,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
index 9dffa94..7b5e4a0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningUtil.java
@@ -28,7 +28,9 @@ public class WarningUtil {
private WarningUtil() {
}
- /** Merges the warnings from the collection argument into the warning collector argument. */
+ /**
+ * Merges the warnings from the collection argument into the warning collector argument.
+ */
public static void mergeWarnings(Collection<Warning> warnings, IWarningCollector warningsCollector) {
for (Warning warning : warnings) {
if (warningsCollector.shouldWarn()) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java
new file mode 100644
index 0000000..c52f27a
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierMappingUtil.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.utils;
+
+import org.apache.asterix.common.api.IIdentifierMapper;
+
+public class IdentifierMappingUtil {
+
+ private static final IIdentifierMapper DEFAULT_MAPPER = identifier -> identifier;
+
+ private static IIdentifierMapper mapper = DEFAULT_MAPPER;
+
+ private IdentifierMappingUtil() {
+ }
+
+ public static void setMapper(IIdentifierMapper mapper) {
+ IdentifierMappingUtil.mapper = mapper;
+ }
+
+ public static String map(String key) {
+ return mapper.map(key);
+ }
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java
new file mode 100644
index 0000000..ebdd740
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IdentifierUtil.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.utils;
+
+public class IdentifierUtil {
+
+ public static final String DATASET = "dataset";
+ public static final String DATAVERSE = "dataverse";
+
+ public static String dataset() {
+ return IdentifierMappingUtil.map(DATASET);
+ }
+
+ public static String dataverse() {
+ return IdentifierMappingUtil.map(DATAVERSE);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index b9ebeb6..8954766 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -163,8 +163,8 @@
1073 = Cannot resolve alias reference for undefined identifier %1$s
1074 = Cannot resolve ambiguous alias reference for identifier %1$s
1075 = Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.
-1076 = Cannot find dataset %1$s because there is no dataverse declared, nor an alias with name %1$s!
-1077 = Cannot find dataset %1$s in dataverse %2$s nor an alias with name %1$s!
+1076 = Cannot find dataset %1$s because there is no dataverse declared, nor an alias with name %1$s
+1077 = Cannot find dataset %1$s in dataverse %2$s nor an alias with name %1$s
1078 = Unexpected operator %1$s in an OperatorExpr starting with %2$s
1079 = Compilation error: %1$s
1080 = Cannot find node group with name %1$s
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
index ea4c6a15..502647e 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_error.md
@@ -71,7 +71,7 @@ cannot be successfully resolved as a valid field access.
If we have a typo as above in "customers" that misses the dataset name's ending "s",
we will get an identifier resolution error as follows:
- ERROR: Code: 1 "ASX1077: Cannot find dataset customer in dataverse Commerce nor an alias with name customer! (in line 2, at column 7)"
+ ERROR: Code: 1 "ASX1077: Cannot find dataset customer in dataverse Commerce nor an alias with name customer (in line 2, at column 7)"
##### Example
(Q4.4)
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 85ab549..3ec8aec 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -19,6 +19,8 @@
package org.apache.asterix.metadata;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -687,7 +689,7 @@ public class MetadataNode implements IMetadataNode {
}
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "node group",
- nodeGroupName, "datasets",
+ nodeGroupName, dataset() + "(s)",
datasets.stream().map(DatasetUtil::getFullyQualifiedDisplayName).collect(Collectors.joining(", ")));
}
try {
@@ -936,7 +938,7 @@ public class MetadataNode implements IMetadataNode {
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
TypeUtil.getFullyQualifiedDisplayName(set.getItemTypeDataverseName(), set.getItemTypeName()),
- "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
+ dataset(), DatasetUtil.getFullyQualifiedDisplayName(set));
}
if (set.getMetaItemTypeDataverseName() != null
&& set.getMetaItemTypeDataverseName().equals(dataverseName)) {
@@ -944,7 +946,7 @@ public class MetadataNode implements IMetadataNode {
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
TypeUtil.getFullyQualifiedDisplayName(set.getMetaItemTypeDataverseName(),
set.getMetaItemTypeName()),
- "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
+ dataset(), DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
@@ -964,9 +966,8 @@ public class MetadataNode implements IMetadataNode {
Function.FunctionDependencyKind functionDependencyKind = functionDependencyKinds[i];
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
- functionDependencyKind.toString().toLowerCase(),
- functionDependencyKind.getDependencyDisplayName(dependency), "function",
- function.getSignature());
+ functionDependencyKind, functionDependencyKind.getDependencyDisplayName(dependency),
+ "function", function.getSignature());
}
}
}
@@ -1098,7 +1099,7 @@ public class MetadataNode implements IMetadataNode {
if (set.getItemTypeName().equals(datatypeName) && set.getItemTypeDataverseName().equals(dataverseName)) {
throw new AsterixException(
org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
- TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), "dataset",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), dataset(),
DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
index 6ba8ba8..3c02a4f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.util.ArrayList;
import java.util.List;
@@ -74,7 +77,7 @@ public class MetadataManagerUtil {
return null;
}
if (dataverseName == null) {
- throw new AlgebricksException("Cannot declare output-record-type with no dataverse!");
+ throw new AlgebricksException("Cannot declare output-record-type with no " + dataverse());
}
IAType type = findType(mdTxnCtx, dataverseName, outputRecordType);
if (!(type instanceof ARecordType)) {
@@ -105,7 +108,8 @@ public class MetadataManagerUtil {
String datasetName) throws AlgebricksException {
Dataset dataset = findDataset(mdTxnCtx, dataverseName, datasetName);
if (dataset == null) {
- throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
+ throw new AlgebricksException(
+ "Unknown " + dataset() + " " + datasetName + " in " + dataverse() + " " + dataverseName);
}
return dataset;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 68def8d..ec63825 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static org.apache.asterix.metadata.utils.MetadataConstants.METADATA_OBJECT_NAME_INVALID_CHARS;
import java.io.File;
@@ -39,6 +41,7 @@ import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.functions.FunctionSignature;
@@ -586,8 +589,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
- throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName());
+ throw new AlgebricksException("Code generation error: no index " + indexName + " for " + dataset() + " "
+ + dataset.getDatasetName());
}
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
@@ -949,8 +952,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
String datasetName = dataSource.getId().getDatasourceName();
Dataset dataset = findDataset(dataSource.getId().getDataverseName(), datasetName);
if (dataset == null) {
- throw new AlgebricksException(
- "Unknown dataset " + datasetName + " in dataverse " + dataSource.getId().getDataverseName());
+ throw new AlgebricksException("Unknown " + dataset() + " " + datasetName + " in " + dataverse() + " "
+ + dataSource.getId().getDataverseName());
}
int numKeys = primaryKeys.size();
int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
@@ -999,7 +1002,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
JobSpecification jobSpec, IAType itemType, ITypedAdapterFactory adapterFactory,
ITupleFilterFactory tupleFilterFactory, long outputLimit) throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
- throw new AlgebricksException("Can only scan datasets of records.");
+ throw new AlgebricksException("Can only scan " + dataset() + "s of records.");
}
ISerializerDeserializer<?> payloadSerde =
@@ -1054,7 +1057,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
keyType = IndexingConstants.getFieldType(j);
break;
default:
- throw new AlgebricksException("Unknown Dataset Type");
+ throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, dsType.toString());
}
} catch (AsterixException e) {
throw new AlgebricksException(e);
@@ -1416,7 +1419,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
// Sanity checks.
if (primaryKeys.size() > 1) {
- throw new AlgebricksException("Cannot create inverted index on dataset with composite primary key.");
+ throw new AlgebricksException(
+ "Cannot create inverted index on " + dataset() + "s with composite primary key.");
}
// The size of secondaryKeys can be two if it receives input from its
// TokenizeOperator- [token, number of token]
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index 3cd4346..a9fa024 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.entities;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -197,5 +199,10 @@ public class Function implements IMetadataEntity<Function> {
public String getDependencyDisplayName(Triple<DataverseName, String, String> dependency) {
return dependencyDisplayNameAccessor.apply(dependency);
}
+
+ @Override
+ public String toString() {
+ return this == DATASET ? dataset() : name().toLowerCase();
+ }
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index d5e941f..92390a7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.feeds;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.rmi.RemoteException;
import java.util.Map;
@@ -53,6 +55,7 @@ import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -73,12 +76,12 @@ public class FeedMetadataUtil {
String datasetName) throws AlgebricksException {
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- throw new CompilationException("Unknown target dataset :" + datasetName);
+ throw new CompilationException("Unknown target " + dataset() + " :" + datasetName);
}
if (!dataset.getDatasetType().equals(DatasetType.INTERNAL)) {
- throw new CompilationException("Statement not applicable. Dataset " + datasetName
- + " is not of required type " + DatasetType.INTERNAL);
+ throw new CompilationException("Statement not applicable. " + StringUtils.capitalize(dataset()) + " "
+ + datasetName + " is not of required type " + DatasetType.INTERNAL);
}
return dataset;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4bcc5f0..4ef349b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.metadata.utils;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+
import java.io.DataOutput;
import java.util.ArrayList;
import java.util.Collections;
@@ -366,7 +369,8 @@ public class DatasetUtil {
DataverseName dataverseName = dataverse.getDataverseName();
Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
if (dataset == null) {
- throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
+ throw new AsterixException(
+ "Could not find " + dataset() + " " + datasetName + " in " + dataverse() + " " + dataverseName);
}
JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
index f2b7558..7a57af2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.utils;
+import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+
import java.util.List;
import java.util.Map;
@@ -75,7 +77,8 @@ public class InvertedIndexResourceFactoryProvider implements IResourceFactoryPro
index.getIndexType().name(), dataset.getDatasetType());
}
if (numPrimaryKeys > 1) {
- throw new AsterixException("Cannot create inverted index on dataset with composite primary key.");
+ throw new AsterixException(
+ "Cannot create inverted index on " + dataset() + "s with composite primary key.");
}
if (numSecondaryKeys > 1) {
throw new AsterixException("Cannot create composite inverted index on multiple fields.");
diff --git a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
index 4b8179c..0b7af91 100644
--- a/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
+++ b/hyracks-fullstack/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
@@ -113,6 +113,7 @@ public class AlgebricksException extends Exception implements IFormattedExceptio
return errorCode;
}
+ @Override
public Serializable[] getParams() {
return params;
}
@@ -121,6 +122,7 @@ public class AlgebricksException extends Exception implements IFormattedExceptio
return nodeId;
}
+ @Override
public SourceLocation getSourceLocation() {
return sourceLoc;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
index 0769e18..977e5d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
@@ -130,6 +130,7 @@ public class HyracksException extends IOException implements IFormattedException
return errorCode;
}
+ @Override
public Serializable[] getParams() {
return params;
}
@@ -138,6 +139,7 @@ public class HyracksException extends IOException implements IFormattedException
return nodeId;
}
+ @Override
public SourceLocation getSourceLocation() {
return sourceLoc;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
index 33b3995..b9d2deb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.api.exceptions;
+import java.io.Serializable;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
@@ -58,6 +59,16 @@ public interface IFormattedException {
Optional<IError> getError();
/**
+ * @return the source location
+ */
+ SourceLocation getSourceLocation();
+
+ /**
+ * @return the parameters to use when formatting
+ */
+ Serializable[] getParams();
+
+ /**
* Indicates whether this exception matches the supplied error code
*/
default boolean matches(IError candidate) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
index ec8f8e9..5f01559 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
@@ -24,26 +24,29 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.Objects;
+import org.apache.commons.lang3.SerializationUtils;
import org.apache.hyracks.api.util.ErrorMessageUtil;
public class Warning implements Serializable {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final String component;
private final SourceLocation srcLocation;
private final int code;
private final String message;
+ private final Serializable[] params;
- private Warning(String component, SourceLocation srcLocation, int code, String message) {
+ private Warning(String component, SourceLocation srcLocation, int code, String message, Serializable... params) {
this.component = component;
this.srcLocation = srcLocation;
this.code = code;
this.message = message;
+ this.params = params;
}
public static Warning of(SourceLocation srcLocation, IError code, Serializable... params) {
return new Warning(code.component(), srcLocation, code.intValue(), ErrorMessageUtil
- .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params));
+ .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params), params);
}
public String getComponent() {
@@ -84,13 +87,25 @@ public class Warning implements Serializable {
output.writeInt(code);
output.writeUTF(message);
SourceLocation.writeFields(srcLocation, output);
+ writeParams(output, params);
+ }
+
+ private static void writeParams(DataOutput output, Serializable[] params) throws IOException {
+ byte[] serialize = SerializationUtils.serialize(params);
+ output.writeInt(serialize.length);
+ output.write(serialize);
}
public static Warning create(DataInput input) throws IOException {
String comp = input.readUTF();
int code = input.readInt();
String msg = input.readUTF();
- return new Warning(comp, SourceLocation.create(input), code, msg);
+ SourceLocation sourceLocation = SourceLocation.create(input);
+ int paramsLen = input.readInt();
+ byte[] paramsBytes = new byte[paramsLen];
+ input.readFully(paramsBytes, 0, paramsBytes.length);
+ Serializable[] params = SerializationUtils.deserialize(paramsBytes);
+ return new Warning(comp, sourceLocation, code, msg, params);
}
@Override
@@ -98,4 +113,8 @@ public class Warning implements Serializable {
return "Warning{" + "component='" + component + '\'' + ", srcLocation=" + srcLocation + ", code=" + code
+ ", message='" + message + '\'' + '}';
}
+
+ public Serializable[] getParams() {
+ return params;
+ }
}
[asterixdb] 19/25: [NO ISSUE][*DB][ACT] +=
ActiveManagerMessage.GENERIC_EVENT
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d18d6ac28b154e1570f863d779b9ae59add6887f
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sat Apr 3 23:12:59 2021 -0400
[NO ISSUE][*DB][ACT] += ActiveManagerMessage.GENERIC_EVENT
Change-Id: I8f8986a90a6ac34a24118ace1a76401d65924055
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10863
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../org/apache/asterix/active/ActiveManager.java | 21 +++++++++++++++++++--
.../org/apache/asterix/active/IActiveRuntime.java | 5 +++++
.../active/message/ActiveManagerMessage.java | 14 +++++++++++---
.../active/message/ActiveStatsRequestMessage.java | 6 +++---
.../active/message/StopRuntimeParameters.java | 12 ++----------
.../app/active/ActiveEntityEventsListener.java | 2 +-
.../apache/asterix/external/api/IRecordReader.java | 5 +++++
.../dataflow/AbstractFeedDataFlowController.java | 5 +++++
.../dataflow/FeedRecordDataFlowController.java | 6 ++++++
.../external/dataset/adapter/FeedAdapter.java | 5 +++++
10 files changed, 62 insertions(+), 19 deletions(-)
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index ba04967..08e1be4 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -112,14 +112,31 @@ public class ActiveManager {
case REQUEST_STATS:
requestStats((ActiveStatsRequestMessage) message);
break;
+ case GENERIC_EVENT:
+ deliverGenericEvent(message);
+ break;
default:
LOGGER.warn("Unknown message type received: " + message.getKind());
}
}
+ private void deliverGenericEvent(ActiveManagerMessage message) throws HyracksDataException {
+ try {
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
+ IActiveRuntime runtime = runtimes.get(runtimeId);
+ if (runtime == null) {
+ LOGGER.warn("Request for a runtime {} that is not registered {}", runtimeId, message);
+ return;
+ }
+ runtime.handleGenericEvent(message);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
private void requestStats(ActiveStatsRequestMessage message) throws HyracksDataException {
try {
- ActiveRuntimeId runtimeId = (ActiveRuntimeId) message.getPayload();
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
IActiveRuntime runtime = runtimes.get(runtimeId);
long reqId = message.getReqId();
if (runtime == null) {
@@ -168,7 +185,7 @@ public class ActiveManager {
@SuppressWarnings("squid:S1181") // Catch Error
private void stopRuntime(ActiveManagerMessage message) {
StopRuntimeParameters content = (StopRuntimeParameters) message.getPayload();
- ActiveRuntimeId runtimeId = content.getRuntimeId();
+ ActiveRuntimeId runtimeId = message.getRuntimeId();
IActiveRuntime runtime = runtimes.get(runtimeId);
if (runtime == null) {
LOGGER.warn("Request to stop runtime: " + runtimeId
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
index a52f01e..b8edc64 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
@@ -20,6 +20,7 @@ package org.apache.asterix.active;
import java.util.concurrent.TimeUnit;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IActiveRuntime {
@@ -48,4 +49,8 @@ public interface IActiveRuntime {
default String getStats() {
return "\"Runtime stats is not available.\"";
}
+
+ default void handleGenericEvent(ActiveManagerMessage event) throws HyracksDataException {
+ throw new IllegalStateException("generic events not supported for runtime " + getRuntimeId());
+ }
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
index 1a2af13..4d726cf 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
@@ -21,6 +21,7 @@ package org.apache.asterix.active.message;
import java.io.Serializable;
import org.apache.asterix.active.ActiveManager;
+import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.messaging.CcIdentifiedMessage;
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
@@ -29,15 +30,18 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public class ActiveManagerMessage extends CcIdentifiedMessage implements INcAddressedMessage {
public enum Kind {
STOP_ACTIVITY,
- REQUEST_STATS
+ REQUEST_STATS,
+ GENERIC_EVENT
}
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final Kind kind;
+ private final ActiveRuntimeId runtimeId;
private final Serializable payload;
- public ActiveManagerMessage(Kind kind, Serializable payload) {
+ public ActiveManagerMessage(Kind kind, ActiveRuntimeId runtimeId, Serializable payload) {
this.kind = kind;
+ this.runtimeId = runtimeId;
this.payload = payload;
}
@@ -45,6 +49,10 @@ public class ActiveManagerMessage extends CcIdentifiedMessage implements INcAddr
return payload;
}
+ public ActiveRuntimeId getRuntimeId() {
+ return runtimeId;
+ }
+
public Kind getKind() {
return kind;
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
index 117a68c..94668a0 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveStatsRequestMessage.java
@@ -18,14 +18,14 @@
*/
package org.apache.asterix.active.message;
-import java.io.Serializable;
+import org.apache.asterix.active.ActiveRuntimeId;
public class ActiveStatsRequestMessage extends ActiveManagerMessage {
private static final long serialVersionUID = 1L;
private final long reqId;
- public ActiveStatsRequestMessage(Serializable payload, long reqId) {
- super(Kind.REQUEST_STATS, payload);
+ public ActiveStatsRequestMessage(ActiveRuntimeId runtimeId, long reqId) {
+ super(Kind.REQUEST_STATS, runtimeId, null);
this.reqId = reqId;
}
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
index fbc41a1..c21f06e 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/StopRuntimeParameters.java
@@ -21,25 +21,17 @@ package org.apache.asterix.active.message;
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
-import org.apache.asterix.active.ActiveRuntimeId;
-
public class StopRuntimeParameters implements Serializable {
- private static final long serialVersionUID = 1L;
- private final ActiveRuntimeId runtimeId;
+ private static final long serialVersionUID = 2L;
private final long timeout;
private final TimeUnit unit;
- public StopRuntimeParameters(ActiveRuntimeId runtimeId, long timeout, TimeUnit unit) {
- this.runtimeId = runtimeId;
+ public StopRuntimeParameters(long timeout, TimeUnit unit) {
this.timeout = timeout;
this.unit = unit;
}
- public ActiveRuntimeId getRuntimeId() {
- return runtimeId;
- }
-
public long getTimeout() {
return timeout;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index c90cde0..5f7d65e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -515,7 +515,7 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl
}
ActiveRuntimeId runtimeId = getActiveRuntimeId(partition++);
messageBroker.sendApplicationMessageToNC(new ActiveManagerMessage(ActiveManagerMessage.Kind.STOP_ACTIVITY,
- new StopRuntimeParameters(runtimeId, timeout, unit)), location);
+ runtimeId, new StopRuntimeParameters(timeout, unit)), location);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
index cb97526..0e6ddb2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
@@ -95,4 +96,8 @@ public interface IRecordReader<T> extends Closeable {
default LongSupplier getLineNumber() {
return ExternalDataConstants.NO_LINES;
}
+
+ default void handleGenericEvent(ActiveManagerMessage event) {
+ throw new IllegalStateException("unexpected generic event " + event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
index b58e604..94d9e6e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.dataflow;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -57,4 +58,8 @@ public abstract class AbstractFeedDataFlowController implements IDataFlowControl
feedLogManager.close();
}
}
+
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ throw new IllegalStateException("unexpected generic event " + event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 56257a8..8cec5de 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.dataflow;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.IRawRecord;
@@ -280,4 +281,9 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
.append("}");
return str.toString();
}
+
+ @Override
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ recordReader.handleGenericEvent(event);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
index 54e633a..fc9b727 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/FeedAdapter.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.dataset.adapter;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -69,4 +70,8 @@ public class FeedAdapter implements IDataSourceAdapter, Closeable {
public void close() throws IOException {
controller.close();
}
+
+ public void handleGenericEvent(ActiveManagerMessage event) {
+ controller.handleGenericEvent(event);
+ }
}
[asterixdb] 25/25: Merge branch 'cheshire-cat' into 'master'
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 21cf9cbe1f228cbd7d161dbd0b23b55403253dd0
Merge: b0be3fc ce5f3aa
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Wed Apr 7 11:52:34 2021 +0300
Merge branch 'cheshire-cat' into 'master'
Change-Id: Ic9ebf963437bf2cac2fb48b43b41728f515133c6
.../org/apache/asterix/active/ActiveManager.java | 21 +-
.../org/apache/asterix/active/IActiveRuntime.java | 5 +
.../active/message/ActiveManagerMessage.java | 14 +-
.../active/message/ActiveStatsRequestMessage.java | 6 +-
.../active/message/StopRuntimeParameters.java | 12 +-
.../operators/physical/InvertedIndexPOperator.java | 4 +-
.../optimizer/rules/ConstantFoldingRule.java | 3 +-
.../rules/MetaFunctionToMetaVariableRule.java | 4 +-
.../rules/SetAsterixPhysicalOperatorsRule.java | 5 +-
.../rules/SweepIllegalNonfunctionalFunctions.java | 3 -
.../subplan/InlineAllNtsInSubplanVisitor.java | 84 +-
...InlineSubplanInputForNestedTupleSourceRule.java | 48 +-
.../rules/subplan/SubplanFlatteningUtil.java | 51 +-
.../rules/util/EquivalenceClassUtils.java | 84 +-
.../asterix/translator/AbstractLangTranslator.java | 21 +-
.../translator/LangExpressionToPlanTranslator.java | 22 +-
.../apache/asterix/api/common/APIFramework.java | 7 +-
.../api/http/server/ConnectorApiServlet.java | 7 +-
.../asterix/api/http/server/NCUdfApiServlet.java | 4 +-
.../api/http/server/QueryServiceServlet.java | 14 +-
.../api/http/server/RebalanceApiServlet.java | 5 +-
.../app/active/ActiveEntityEventsListener.java | 2 +-
.../asterix/app/function/DatasetRewriter.java | 6 +-
.../apache/asterix/app/function/FeedRewriter.java | 5 +-
.../app/message/StorageCleanupRequestMessage.java | 109 +
.../apache/asterix/app/message/VoidResponse.java | 63 +
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +-
.../org/apache/asterix/app/nc/RecoveryManager.java | 19 +-
.../asterix/app/translator/QueryTranslator.java | 43 +-
.../hyracks/bootstrap/GlobalRecoveryManager.java | 31 +
.../test/dataflow/GlobalStorageCleanupTest.java | 68 +
.../test/dataflow/LSMFlushRecoveryTest.java | 12 +-
.../src/test/resources/TweetSent/roundtrip.py | 4 +
asterixdb/asterix-app/src/test/resources/cc.conf | 1 +
.../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 +
.../queries/subquery/in_let_8.sqlpp} | 43 +-
.../queries/subquery/query-ASTERIXDB-2845.sqlpp | 2 +-
.../optimizerts/results/ASTERIXDB-2402.plan | 28 +-
.../leftouterjoin/query-ASTERIXDB-2857.plan | 39 +
.../optimizerts/results/query-issue562_ps.plan | 58 +-
.../optimizerts/results/subquery/exists.plan | 2 +-
.../optimizerts/results/subquery/exists_ps.plan | 4 +-
.../optimizerts/results/subquery/in_let_3.plan | 4 +-
.../optimizerts/results/subquery/in_let_4.plan | 4 +-
.../optimizerts/results/subquery/in_let_5.plan | 4 +-
.../subquery/{in_let_3.plan => in_let_8.plan} | 60 +-
.../optimizerts/results/subquery/not_exists.plan | 2 +-
.../results/subquery/not_exists_ps.plan | 4 +-
.../results/subquery/query-ASTERIXDB-2845.plan | 145 +-
.../tpcds/query-ASTERIXDB-1581-correlated-2.plan | 399 +-
.../tpcds/query-ASTERIXDB-1581-correlated.plan | 407 +-
.../external_dataset.000.ddl.sqlpp | 66 +
.../external_dataset.001.query.sqlpp} | 7 +-
.../external_dataset.002.query.sqlpp} | 7 +-
.../external_dataset.003.query.sqlpp} | 7 +-
.../external_dataset.004.query.sqlpp} | 7 +-
.../external_dataset.005.query.sqlpp} | 7 +-
.../external_dataset.006.query.sqlpp} | 7 +-
.../external_dataset.999.ddl.sqlpp} | 6 +-
.../py_function_error.2.ddl.sqlpp | 5 +-
...2.ddl.sqlpp => py_function_error.5.query.sqlpp} | 12 +-
...2.ddl.sqlpp => py_function_error.6.query.sqlpp} | 12 +-
.../query-ASTERIXDB-2857.1.ddl.sqlpp} | 30 +-
.../query-ASTERIXDB-2857.2.update.sqlpp} | 6 +-
.../query-ASTERIXDB-2857.3.query.sqlpp} | 23 +-
.../in_let/in_let.8.query.sqlpp} | 21 +-
.../synonym-03/synonym-03.1.ddl.sqlpp} | 19 +-
.../synonym-03/synonym-03.2.query.sqlpp} | 9 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +
.../cluster_state_1_full.1.regexadm | 2 +
.../cluster_state_1_less.1.regexadm | 2 +
.../results/binary/query_id/query_id.1.adm | 6005 --------------------
.../custom-buffer-size/external_dataset.001.adm | 1 +
.../custom-buffer-size/external_dataset.002.adm | 1 +
.../custom-buffer-size/external_dataset.003.adm | 1 +
.../custom-buffer-size/external_dataset.004.adm | 25 +
.../custom-buffer-size/external_dataset.005.adm | 1 +
.../custom-buffer-size/external_dataset.006.adm | 25 +
.../py_function_error/py_function_error.3.json | 1 +
.../py_function_error/py_function_error.4.json | 1 +
.../query-ASTERIXDB-2857.3.adm | 2 +
.../special_chars_2/special_chars_2.2.adm | 6 +-
.../runtimets/results/subquery/in_let/in_let.8.adm | 2 +
.../results/synonym/synonym-03/synonym-03.2.adm | 2 +
.../resources/runtimets/testsuite_it_python.xml | 2 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 41 +-
.../asterix/common/api/IIdentifierMapper.java} | 11 +-
.../asterix/common/config/CompilerProperties.java | 17 +-
.../asterix/common/config/ExternalProperties.java | 8 +-
.../common/config/OptimizationConfUtil.java | 15 +
.../asterix/common/config/StorageProperties.java | 54 +-
.../common/config/TransactionProperties.java | 5 +-
.../asterix/common/exceptions/WarningUtil.java | 4 +-
.../asterix/common/metadata/DataverseName.java | 74 +-
.../common/utils/IdentifierMappingUtil.java} | 25 +-
.../asterix/common/utils/IdentifierUtil.java} | 18 +-
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../asterix/common/metadata/DataverseNameTest.java | 25 +-
.../asterix-doc/src/main/markdown/sqlpp/5_error.md | 2 +-
.../src/main/markdown/sqlpp/7_ddl_dml.md | 3 +-
.../apache/asterix/external/api/IRecordReader.java | 5 +
.../dataflow/AbstractFeedDataFlowController.java | 5 +
.../dataflow/FeedRecordDataFlowController.java | 6 +
.../external/dataset/adapter/FeedAdapter.java | 5 +
.../abstracts/AbstractExternalInputStream.java | 3 -
.../AbstractExternalInputStreamFactory.java | 32 +-
.../input/record/reader/aws/AwsS3InputStream.java | 82 +-
.../record/reader/aws/AwsS3InputStreamFactory.java | 30 +-
.../record/reader/azure/AzureBlobInputStream.java | 12 +-
.../reader/azure/AzureBlobInputStreamFactory.java | 30 +-
.../record/reader/stream/StreamRecordReader.java | 6 +-
.../input/stream/AsterixInputStreamReader.java | 11 +-
.../StandardUTF8ToModifiedUTF8DataOutput.java | 2 +-
.../external/library/PythonLibraryEvaluator.java | 17 +-
.../library/PythonLibraryEvaluatorFactory.java | 34 +-
.../external/util/ExternalDataConstants.java | 14 +-
.../asterix/external/util/ExternalDataUtils.java | 13 +-
.../input/record/reader/awss3/AwsS3Test.java | 145 +
.../org/apache/asterix/metadata/MetadataNode.java | 14 +-
.../metadata/declared/DatasetDataSource.java | 6 +
.../metadata/declared/MetadataManagerUtil.java | 8 +-
.../metadata/declared/MetadataProvider.java | 17 +-
.../apache/asterix/metadata/entities/Function.java | 7 +
.../asterix/metadata/feeds/FeedMetadataUtil.java | 9 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 6 +-
.../InvertedIndexResourceFactoryProvider.java | 5 +-
.../asterix/om/functions/BuiltinFunctions.java | 2 +-
.../replication/management/NetworkingUtil.java | 16 +-
.../replication/messaging/ReplicateFileTask.java | 7 +
.../asterix/replication/sync/FileSynchronizer.java | 4 +
.../functions/CreateQueryUIDDescriptor.java | 32 +-
.../service/transaction/TransactionManager.java | 12 +-
asterixdb/pom.xml | 2 +-
.../appended-resources/supplemental-models.xml | 32 +-
.../common/exceptions/AlgebricksException.java | 2 +
.../api/HeuristicCompilerFactoryBuilder.java | 3 +-
.../algebricks/algebricks-core/pom.xml | 2 +-
.../logical/RunningAggregateOperator.java | 5 +
.../algebra/util/OperatorManipulationUtil.java | 9 +-
.../algebricks/core/config/AlgebricksConfig.java | 3 +
.../algebricks/core/jobgen/impl/JobGenContext.java | 9 +-
.../rewriter/base/PhysicalOptimizationConfig.java | 9 +
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 +-
.../hyracks/api/exceptions/HyracksException.java | 2 +
.../api/exceptions/IFormattedException.java | 11 +
.../org/apache/hyracks/api/exceptions/Warning.java | 27 +-
.../org/apache/hyracks/api/util/InvokeUtil.java | 2 +-
.../control/common/controllers/NCConfig.java | 4 +
.../hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +-
.../hyracks/dataflow/common/io/RunFileWriter.java | 10 +-
.../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 +-
.../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +-
.../hyracks/http/server/AbstractServlet.java | 9 +-
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 2 +-
hyracks-fullstack/pom.xml | 4 +-
155 files changed, 2427 insertions(+), 7219 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index b83fbdf,919a24e..ccd38d3
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@@ -134,9 -135,10 +135,10 @@@ public class APIFramework
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY,
+ CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
- CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
+ CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ce6e10d,d8698f9..d2e9284
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -18,7 -18,8 +18,9 @@@
*/
package org.apache.asterix.app.translator;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
import java.io.File;
import java.io.FileInputStream;
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 8064c05,a58673f..5538a0a
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@@ -160,8 -165,8 +169,10 @@@ public class CompilerProperties extend
public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
+ public static final String COMPILER_ARRAYINDEX_KEY = Option.COMPILER_ARRAYINDEX.ini();
+
+ public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@@ -228,7 -233,7 +239,11 @@@
return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
}
+ public boolean isArrayIndex() {
+ return accessor.getBoolean(Option.COMPILER_ARRAYINDEX);
+ }
++
+ public int getExternalScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
+ }
}
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 81974fa,c4dbd85..d498d69
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@@ -72,9 -72,9 +72,12 @@@ public class OptimizationConfUtil
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
boolean minMemoryAllocation = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
+ boolean arrayIndex = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
+ compilerProperties.isArrayIndex());
+ int externalScanBufferSize = getExternalScanBufferSize(
+ (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
+ compilerProperties.getExternalScanMemorySize(), sourceLoc);
+
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
@@@ -90,7 -90,7 +93,8 @@@
physOptConf.setSubplanMerge(subplanMerge);
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
+ physOptConf.setArrayIndexEnabled(arrayIndex);
+ physOptConf.setExternalScanBufferSize(externalScanBufferSize);
return physOptConf;
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 09709d0,3ec8aec..c769960
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@@ -19,7 -19,7 +19,8 @@@
package org.apache.asterix.metadata;
+import static org.apache.asterix.common.exceptions.ErrorCode.FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.rmi.RemoteException;
import java.util.ArrayList;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9241458,ec63825..497b15f
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@@ -618,10 -589,9 +620,10 @@@ public class MetadataProvider implement
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
- throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName());
+ throw new AlgebricksException("Code generation error: no index " + indexName + " for " + dataset() + " "
+ + dataset.getDatasetName());
}
+ Index.ValueIndexDetails secondaryIndexDetails = (Index.ValueIndexDetails) secondaryIndex.getIndexDetails();
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
getSplitProviderAndConstraints(dataset, secondaryIndex.getIndexName());
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 944ddf7,817db5a..20632f1
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@@ -33,5 -34,6 +34,7 @@@ public class AlgebricksConfig
public static final boolean SUBPLAN_MERGE_DEFAULT = true;
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
+ public static final boolean ARRAY_INDEX_DEFAULT = false;
+ public static final int EXTERNAL_SCAN_BUFFER_SIZE =
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
}
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index f18cad2,bfff925..62bf671
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@@ -45,7 -45,7 +45,8 @@@ public class PhysicalOptimizationConfi
private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
+ private static final String ARRAY_INDEX = "ARRAY_INDEX";
+ private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
private final Properties properties = new Properties();
@@@ -227,14 -227,14 +228,22 @@@
setBoolean(MIN_MEMORY_ALLOCATION, value);
}
+ public boolean isArrayIndexEnabled() {
+ return getBoolean(ARRAY_INDEX, AlgebricksConfig.ARRAY_INDEX_DEFAULT);
+ }
+
+ public void setArrayIndexEnabled(boolean arrayIndex) {
+ setBoolean(ARRAY_INDEX, arrayIndex);
+ }
+
+ public int getExternalScanBufferSize() {
+ return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
+ }
+
+ public void setExternalScanBufferSize(int bufferSize) {
+ setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
[asterixdb] 14/25: [NO ISSUE][COMP] Eliminate unused running
aggregates
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 778db56944e04bfa93ab91ab7709537d0dbe5d8d
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Wed Mar 31 14:58:42 2021 -0700
[NO ISSUE][COMP] Eliminate unused running aggregates
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Improve RemoveUnusedAssignAndAggregateRule
to eliminate unused running aggregates
Change-Id: Iaf002dcb2aa56946df97fb08738d4c03ed308d7c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10804
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../results/leftouterjoin/loj-03-no-listify.plan | 134 ++++++++++-----------
.../optimizerts/results/query_issue849.plan | 8 +-
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++-
3 files changed, 82 insertions(+), 75 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
index 646ab5b..6d569dc 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/loj-03-no-listify.plan
@@ -10,87 +10,54 @@
-- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- RUNNING_AGGREGATE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- RUNNING_AGGREGATE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
- -- HASH_PARTITION_MERGE_EXCHANGE MERGE:[$$taskId(ASC)] HASH:[$$taskId] |PARTITIONED|
- -- SORT_GROUP_BY[$$279] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$279] |PARTITIONED|
- -- SORT_GROUP_BY[$$242] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$taskId][$$taskId] |PARTITIONED|
+ -- HASH_PARTITION_MERGE_EXCHANGE MERGE:[$$taskId(ASC)] HASH:[$$taskId] |PARTITIONED|
+ -- SORT_GROUP_BY[$$279] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$279] |PARTITIONED|
+ -- SORT_GROUP_BY[$$242] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$281] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$281] |PARTITIONED|
- -- SORT_GROUP_BY[$$243] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- REPLICATE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- SORT_GROUP_BY[$$283] |PARTITIONED|
- {
- -- AGGREGATE |LOCAL|
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- HASH_PARTITION_EXCHANGE [$$283] |PARTITIONED|
- -- SORT_GROUP_BY[$$244] |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$281] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- STREAM_SELECT |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$281] |PARTITIONED|
+ -- SORT_GROUP_BY[$$243] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- REPLICATE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -100,6 +67,35 @@
-- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$283] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- HASH_PARTITION_EXCHANGE [$$283] |PARTITIONED|
+ -- SORT_GROUP_BY[$$244] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.tasks) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ASSIGN |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
index fd70464..5d16539 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
@@ -22,11 +22,9 @@
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- HYBRID_HASH_JOIN [$$45][$$44] |PARTITIONED|
-- HASH_PARTITION_EXCHANGE [$$45] |PARTITIONED|
- -- STREAM_PROJECT |UNPARTITIONED|
- -- RUNNING_AGGREGATE |UNPARTITIONED|
- -- ASSIGN |UNPARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- ASSIGN |UNPARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index a563f46..956f0b7 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -43,6 +43,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperator;
@@ -50,7 +51,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.Var
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/**
- * Removes unused variables from Assign, Unnest, Aggregate, UnionAll, and Group-by operators.
+ * Removes unused variables from Assign, Unnest, Aggregate, RunningAggregate, UnionAll, and Group-by operators.
*/
public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
@@ -223,6 +224,13 @@ public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule
isTransformed = true;
}
return agg.getVariables().size();
+ case RUNNINGAGGREGATE:
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ if (removeUnusedVarsAndExprs(toRemove, ragg.getVariables(), ragg.getExpressions())) {
+ context.computeAndSetTypeEnvironmentForOperator(ragg);
+ isTransformed = true;
+ }
+ return ragg.getVariables().size();
case UNNEST:
UnnestOperator uOp = (UnnestOperator) op;
LogicalVariable pVar = uOp.getPositionalVariable();
@@ -354,6 +362,11 @@ public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule
assignVarsSetInThisOp.addAll(agg.getVariables());
targetOpFound = true;
break;
+ case RUNNINGAGGREGATE:
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ assignVarsSetInThisOp.addAll(ragg.getVariables());
+ targetOpFound = true;
+ break;
case UNNEST:
UnnestOperator uOp = (UnnestOperator) op;
LogicalVariable pVar = uOp.getPositionalVariable();
[asterixdb] 24/25: Merge mad-hatter into cheshire-cat
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ce5f3aa366421a1106c842c3a818e283fe050f3d
Merge: 5b5ac48 3ea8ef1
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Apr 7 00:13:08 2021 +0300
Merge mad-hatter into cheshire-cat
Change-Id: I6e2ad0d4cf85f8706063db84b7af9fd4f6c83843
.../optimizer/rules/ConstantFoldingRule.java | 3 +-
.../apache/asterix/api/common/APIFramework.java | 3 +-
.../external_dataset.000.ddl.sqlpp | 66 ++++++++++++++++++++++
.../external_dataset.001.query.sqlpp | 23 ++++++++
.../external_dataset.002.query.sqlpp | 23 ++++++++
.../external_dataset.003.query.sqlpp | 23 ++++++++
.../external_dataset.004.query.sqlpp | 23 ++++++++
.../external_dataset.005.query.sqlpp | 23 ++++++++
.../external_dataset.006.query.sqlpp | 23 ++++++++
.../external_dataset.999.ddl.sqlpp | 20 +++++++
.../custom-buffer-size/external_dataset.001.adm | 1 +
.../custom-buffer-size/external_dataset.002.adm | 1 +
.../custom-buffer-size/external_dataset.003.adm | 1 +
.../custom-buffer-size/external_dataset.004.adm | 25 ++++++++
.../custom-buffer-size/external_dataset.005.adm | 1 +
.../custom-buffer-size/external_dataset.006.adm | 25 ++++++++
.../asterix/common/config/CompilerProperties.java | 15 +++++
.../common/config/OptimizationConfUtil.java | 16 +++++-
.../input/record/reader/aws/AwsS3InputStream.java | 5 +-
.../record/reader/stream/StreamRecordReader.java | 6 +-
.../input/stream/AsterixInputStreamReader.java | 11 ++--
.../StandardUTF8ToModifiedUTF8DataOutput.java | 2 +-
.../external/util/ExternalDataConstants.java | 1 +
.../asterix/external/util/ExternalDataUtils.java | 6 ++
.../metadata/declared/DatasetDataSource.java | 6 ++
.../api/HeuristicCompilerFactoryBuilder.java | 3 +-
.../algebricks/core/config/AlgebricksConfig.java | 3 +
.../algebricks/core/jobgen/impl/JobGenContext.java | 9 ++-
.../rewriter/base/PhysicalOptimizationConfig.java | 9 +++
29 files changed, 357 insertions(+), 19 deletions(-)
diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index e99817d,6f359c5..0c2eeba
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@@ -147,11 -144,10 +148,11 @@@ public class ConstantFoldingRule implem
jobGenCtx = new JobGenContext(null, metadataProvider, appCtx, SerializerDeserializerProvider.INSTANCE,
BinaryHashFunctionFactoryProvider.INSTANCE, BinaryHashFunctionFamilyProvider.INSTANCE,
BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY,
- BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null,
+ BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE,
+ UnnestingPositionWriterFactory.INSTANCE, null,
new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null,
- NoOpWarningCollector.INSTANCE, 0);
+ NoOpWarningCollector.INSTANCE, 0, new PhysicalOptimizationConfig());
}
@Override
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 0091304,c863080..919a24e
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@@ -130,22 -129,18 +130,23 @@@ public class APIFramework
public static final String PREFIX_INTERNAL_PARAMETERS = "_internal";
// A white list of supported configurable parameters.
- private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
- ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
- CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
- CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
- CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
- FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
- StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
- FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
- SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
- AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION,
- DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION);
+ private static final Set<String> CONFIGURABLE_PARAMETER_NAMES = ImmutableSet.of(
+ CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
+ CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
+ CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
+ CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
++ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY,
++ CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
+ CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
+ CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
+ CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
+ StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
+ FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
+ SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
+ DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
+ SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
+ EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
index 0000000,22a30ca..22a30ca
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
index 0000000,7c1708a..7c1708a
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
index 0000000,d122f42..d122f42
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
index 0000000,552f943..552f943
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
index 0000000,f30af80..f30af80
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
index 0000000,af6aff0..af6aff0
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
index 0000000,854ac80..854ac80
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
index 0000000,548e632..548e632
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
index 0000000,187a8cb..187a8cb
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
index 0000000,187a8cb..187a8cb
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
index 0000000,5db606c..5db606c
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
index 0000000,7660e7e..7660e7e
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
index 0000000,5db606c..5db606c
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
index 0000000,7643986..7643986
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 866d183,652f390..a58673f
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@@ -55,6 -56,10 +55,10 @@@ public class CompilerProperties extend
LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(32L, MEGABYTE),
"The memory budget (in bytes) for an inverted-index-search operator instance in a partition"),
+ COMPILER_EXTERNALSCANMEMORY(
+ INTEGER_BYTE_UNIT,
- StorageUtil.getIntSizeInBytes(4, KILOBYTE),
++ StorageUtil.getIntSizeInBytes(8, KILOBYTE),
+ "The memory budget (in bytes) for an external scan operator instance in a partition"),
COMPILER_FRAMESIZE(
INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
@@@ -126,6 -111,11 +130,11 @@@
public Object defaultValue() {
return defaultValue;
}
+
+ @Override
+ public boolean hidden() {
- return this == COMPILER_STRINGOFFSET || this == COMPILER_EXTERNALSCANMEMORY;
++ return this == COMPILER_EXTERNALSCANMEMORY;
+ }
}
public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini();
@@@ -144,18 -134,8 +153,20 @@@
public static final String COMPILER_SORT_SAMPLES_KEY = Option.COMPILER_SORT_SAMPLES.ini();
+ public static final String COMPILER_INDEXONLY_KEY = Option.COMPILER_INDEXONLY.ini();
+
+ public static final String COMPILER_INTERNAL_SANITYCHECK_KEY = Option.COMPILER_INTERNAL_SANITYCHECK.ini();
+
+ public static final String COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY = Option.COMPILER_EXTERNAL_FIELD_PUSHDOWN.ini();
+
+ public static final String COMPILER_SUBPLAN_MERGE_KEY = Option.COMPILER_SUBPLAN_MERGE.ini();
+
+ public static final String COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY = Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN.ini();
+
+ public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
+
+ public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@@ -198,27 -183,7 +209,31 @@@
return accessor.getInt(Option.COMPILER_SORT_SAMPLES);
}
+ public boolean isIndexOnly() {
+ return accessor.getBoolean(Option.COMPILER_INDEXONLY);
+ }
+
+ public boolean isSanityCheck() {
+ return accessor.getBoolean(Option.COMPILER_INTERNAL_SANITYCHECK);
+ }
+
+ public boolean isFieldAccessPushdown() {
+ return accessor.getBoolean(Option.COMPILER_EXTERNAL_FIELD_PUSHDOWN);
+ }
+
+ public boolean getSubplanMerge() {
+ return accessor.getBoolean(Option.COMPILER_SUBPLAN_MERGE);
+ }
+
+ public boolean getSubplanNestedPushdown() {
+ return accessor.getBoolean(Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN);
+ }
+
+ public boolean getMinMemoryAllocation() {
+ return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
+ }
++
+ public int getExternalScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
+ }
}
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 8832054,22fb13b..c4dbd85
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@@ -58,21 -58,11 +58,23 @@@ public class OptimizationConfUtil
compilerProperties.getWindowMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_WINDOW, sourceLoc);
int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc);
int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc);
- boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig);
+ boolean fullParallelSort = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
+ compilerProperties.getSortParallel());
+ boolean indexOnly = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_INDEXONLY_KEY,
+ compilerProperties.isIndexOnly());
+ boolean sanityCheck = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
+ compilerProperties.isSanityCheck());
+ boolean externalFieldPushdown = getBoolean(querySpecificConfig,
+ CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, compilerProperties.isFieldAccessPushdown());
+ boolean subplanMerge = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
+ compilerProperties.getSubplanMerge());
+ boolean subplanNestedPushdown = getBoolean(querySpecificConfig,
+ CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
+ boolean minMemoryAllocation = getBoolean(querySpecificConfig,
+ CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
-
+ int externalScanBufferSize = getExternalScanBufferSize(
+ (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
+ compilerProperties.getExternalScanMemorySize(), sourceLoc);
-
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
@@@ -82,12 -72,8 +84,13 @@@
physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
physOptConf.setSortParallel(fullParallelSort);
physOptConf.setSortSamples(sortNumSamples);
+ physOptConf.setIndexOnly(indexOnly);
+ physOptConf.setSanityCheckEnabled(sanityCheck);
+ physOptConf.setExternalFieldPushdown(externalFieldPushdown);
+ physOptConf.setSubplanMerge(subplanMerge);
+ physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
+ physOptConf.setMinMemoryAllocation(minMemoryAllocation);
+ physOptConf.setExternalScanBufferSize(externalScanBufferSize);
-
return physOptConf;
}
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 4288cc4,8bd7a51..5eb8475
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@@ -41,35 -41,71 +41,34 @@@ import software.amazon.awssdk.core.exce
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
-public class AwsS3InputStream extends AbstractMultipleInputStream {
-
- private static final Logger LOGGER = LogManager.getLogger();
+public class AwsS3InputStream extends AbstractExternalInputStream {
- private final S3Client s3Client;
+ // Configuration
private final String bucket;
- private final int bufferSize;
-
+ private final S3Client s3Client;
-
- // File fields
- private final List<String> filePaths;
- private int nextFileIndex = 0;
+ private static final int MAX_RETRIES = 5; // We will retry 5 times in case of internal error from AWS S3 service
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
- this.filePaths = filePaths;
+ super(configuration, filePaths);
this.s3Client = buildAwsS3Client(configuration);
- this.bufferSize = ExternalDataUtils.getOrDefaultBufferSize(configuration);
- this.bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ this.bucket = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@Override
- protected boolean advance() throws IOException {
- // No files to read for this partition
- if (filePaths == null || filePaths.isEmpty()) {
- return false;
- }
-
- // Finished reading all the files
- if (nextFileIndex >= filePaths.size()) {
- return false;
- }
-
- // Close the current stream before going to the next one
- if (in != null) {
- CleanupUtils.close(in, null);
- }
-
+ protected boolean getInputStream() throws IOException {
String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
- GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(fileName).build();
-
+ GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(filePaths.get(nextFileIndex)).build();
-
// Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
// the header, then the S3 stream gets closed in the close method
- try {
- in = s3Client.getObject(getObjectRequest);
- } catch (NoSuchKeyException ex) {
- LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket "
- + getObjectRequest.bucket());
- nextFileIndex++;
- return advance();
- } catch (SdkException ex) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ if (!doGetInputStream(getObjectRequest)) {
+ return false;
}
--
// Use gzip stream if needed
if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
- in = new GZIPInputStream(in, bufferSize);
- }
-
- // Current file ready, point to the next file
- nextFileIndex++;
- if (notificationHandler != null) {
- notificationHandler.notifyNewSource();
+ in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
return true;
}
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index d2499f8,ddbd350..81da0b0
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@@ -33,19 -32,19 +32,19 @@@ import org.apache.hyracks.api.exception
public class AsterixInputStreamReader extends Reader {
private final AsterixInputStream in;
- private final byte[] bytes;
- private final ByteBuffer byteBuffer;
- private final CharBuffer charBuffer;
private final CharsetDecoder decoder;
- private final byte[] bytes;
- private boolean done = false;
- private boolean remaining = false;
++ private byte[] bytes;
+ protected final ByteBuffer byteBuffer;
+ protected final CharBuffer charBuffer;
+ protected boolean done = false;
+ protected boolean remaining = false;
- public AsterixInputStreamReader(AsterixInputStream in) {
+ public AsterixInputStreamReader(AsterixInputStream in, int bufferSize) {
this.in = in;
+ this.decoder = StandardCharsets.UTF_8.newDecoder();
- bytes = new byte[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
- byteBuffer = ByteBuffer.wrap(bytes);
- charBuffer = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ this.bytes = new byte[bufferSize];
+ this.byteBuffer = ByteBuffer.wrap(bytes);
+ this.charBuffer = CharBuffer.allocate(bufferSize);
- this.decoder = StandardCharsets.UTF_8.newDecoder();
this.byteBuffer.flip();
}
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
index be0d286,0000000..adeb80e
mode 100644,000000..100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/StandardUTF8ToModifiedUTF8DataOutput.java
@@@ -1,158 -1,0 +1,158 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.stream;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+
+/**
+ * Writes modified UTF-8 string format to {@link StandardUTF8ToModifiedUTF8DataOutput#out}
+ * from standard UTF-8 string format.
+ */
+public class StandardUTF8ToModifiedUTF8DataOutput implements DataOutput {
+ private static final byte[] EMPTY = new byte[0];
+ private final AStringSerializerDeserializer stringSerDer;
+ private final ResettableUTF8InputStreamReader reader;
+ private final char[] inputBuffer;
+ private char[] appendBuffer;
+ private DataOutput out;
+
+ public StandardUTF8ToModifiedUTF8DataOutput(AStringSerializerDeserializer stringSerDer) {
+ this.stringSerDer = stringSerDer;
+ reader = new ResettableUTF8InputStreamReader(new ByteArrayAccessibleInputStream(EMPTY, 0, 0));
+ inputBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ appendBuffer = new char[ExternalDataConstants.DEFAULT_BUFFER_SIZE];
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ reader.prepareNextRead(b, off, len);
+ int numOfChars = reader.read(inputBuffer);
+ int length = 0;
+ while (numOfChars > 0) {
+ appendBuffer = append(inputBuffer, appendBuffer, length, numOfChars);
+ length += numOfChars;
+ numOfChars = reader.read(inputBuffer);
+ }
+ stringSerDer.serialize(appendBuffer, 0, length, out);
+ }
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ public void setDataOutput(DataOutput out) {
+ this.out = out;
+ }
+
+ private static char[] append(char[] src, char[] dest, int offset, int length) {
+ char[] destBuf = dest;
+ if (offset + length > dest.length) {
+ char[] newDestBuffer = new char[dest.length * 2];
+ System.arraycopy(destBuf, 0, newDestBuffer, 0, offset);
+ destBuf = newDestBuffer;
+ }
+ System.arraycopy(src, 0, destBuf, offset, length);
+ return destBuf;
+ }
+
+ private static class ResettableUTF8InputStreamReader extends AsterixInputStreamReader {
+ private final ByteArrayAccessibleInputStream inByte;
+
+ public ResettableUTF8InputStreamReader(ByteArrayAccessibleInputStream inByte) {
- super(new BasicInputStream(inByte));
++ super(new BasicInputStream(inByte), ExternalDataConstants.DEFAULT_BUFFER_SIZE);
+ this.inByte = inByte;
+ }
+
+ //Rewind the reader after setting the byte array
+ public void prepareNextRead(byte[] b, int off, int len) {
+ inByte.setContent(b, off, len);
+ done = false;
+ remaining = false;
+ byteBuffer.flip();
+ charBuffer.flip();
+ }
+
+ }
+}
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 047fe2a,1911083..a0cf387
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@@ -119,10 -117,10 +119,11 @@@ public class ExternalDataConstants
// a string representing the NULL value
public static final String KEY_NULL_STR = "null";
public static final String KEY_REDACT_WARNINGS = "redact-warnings";
+ public static final String KEY_REQUESTED_FIELDS = "requested-fields";
+ public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
/**
- * Keys for adapter name
+ * Keys for adapter name
**/
public static final String KEY_ADAPTER_NAME_TWITTER_PUSH = "twitter_push";
public static final String KEY_ADAPTER_NAME_PUSH_TWITTER = "push_twitter";
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 2d25d08,bc9a038..4beb932
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@@ -18,20 -18,9 +18,21 @@@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_KEY;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_BLOB_ENDPOINT;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ENDPOINT_SUFFIX;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_SHARED_ACCESS_SIGNATURE;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
+ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 29a66b9,715092a..17f91b4
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@@ -18,7 -18,8 +18,9 @@@
*/
package org.apache.asterix.metadata.declared;
+ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
+
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@@ -109,14 -115,16 +112,17 @@@ public class DatasetDataSource extends
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
+ PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+ int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
- Map<String, String> properties = edd.getProperties();
+ Map<String, String> properties = addProjectionInfo(projectionInfo, edd.getProperties());
+ properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
+ return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
+ tupleFilterFactory, outputLimit);
case INTERNAL:
DataSourceId id = getId();
- String dataverseName = id.getDataverseName();
+ DataverseName dataverseName = id.getDataverseName();
String datasetName = id.getDatasourceName();
Index primaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName, datasetName, datasetName);
diff --cc hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index 49be31c,b96d657..c6a79a5
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@@ -102,12 -102,11 +102,11 @@@ public class HeuristicCompilerFactoryBu
serializerDeserializerProvider, hashFunctionFactoryProvider, hashFunctionFamilyProvider,
comparatorFactoryProvider, typeTraitProvider, binaryBooleanInspectorFactory,
binaryIntegerInspectorFactory, printerProvider, missingWriterFactory,
- normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer,
- oc, expressionEvalSizeComputer, partialAggregationTypeComputer,
- predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(),
- clusterLocations, warningCollector, maxWarnings, physicalOptimizationConfig);
-
+ unnestingPositionWriterFactory, normalizedKeyComputerFactoryProvider,
+ expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer,
+ partialAggregationTypeComputer, predEvaluatorFactoryProvider,
+ physicalOptimizationConfig.getFrameSize(), clusterLocations, warningCollector,
- maxWarnings);
-
++ maxWarnings, physicalOptimizationConfig);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
}
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index c99117b,7630a13..817db5a
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@@ -25,12 -26,8 +26,14 @@@ public class AlgebricksConfig
public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
- public static final int SORT_SAMPLES = 100;
+ public static final int SORT_SAMPLES_DEFAULT = 100;
+ public static final boolean SORT_PARALLEL_DEFAULT = true;
+ public static final boolean INDEX_ONLY_DEFAULT = true;
+ public static final boolean SANITYCHECK_DEFAULT = false;
+ public static final boolean EXTERNAL_FIELD_PUSHDOWN_DEFAULT = false;
+ public static final boolean SUBPLAN_MERGE_DEFAULT = true;
+ public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
+ public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
+ public static final int EXTERNAL_SCAN_BUFFER_SIZE =
- StorageUtil.getIntSizeInBytes(4, StorageUtil.StorageUnit.KILOBYTE);
- public static final boolean SORT_PARALLEL = true;
++ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
}
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index 92f579c,93f5d74..bfff925
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@@ -39,14 -39,9 +39,15 @@@ public class PhysicalOptimizationConfi
private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
private static final String SORT_PARALLEL = "SORT_PARALLEL";
private static final String SORT_SAMPLES = "SORT_SAMPLES";
+ private static final String INDEX_ONLY = "INDEX_ONLY";
+ private static final String SANITY_CHECK = "SANITY_CHECK";
+ private static final String EXTERNAL_FIELD_PUSHDOWN = "EXTERNAL_FIELD_PUSHDOWN";
+ private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
+ private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
+ private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
+ private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
- private Properties properties = new Properties();
+ private final Properties properties = new Properties();
public PhysicalOptimizationConfig() {
int frameSize = 32768;
@@@ -178,54 -173,14 +179,62 @@@
setInt(SORT_SAMPLES, sortSamples);
}
+ public void setIndexOnly(boolean indexOnly) {
+ setBoolean(INDEX_ONLY, indexOnly);
+ }
+
+ public boolean isIndexOnly() {
+ return getBoolean(INDEX_ONLY, AlgebricksConfig.INDEX_ONLY_DEFAULT);
+ }
+
+ public void setSanityCheckEnabled(boolean sanityCheck) {
+ setBoolean(SANITY_CHECK, sanityCheck);
+ }
+
+ public boolean isSanityCheckEnabled() {
+ return getBoolean(SANITY_CHECK, AlgebricksConfig.SANITYCHECK_DEFAULT);
+ }
+
+ public boolean isExternalFieldPushdown() {
+ return getBoolean(EXTERNAL_FIELD_PUSHDOWN, AlgebricksConfig.EXTERNAL_FIELD_PUSHDOWN_DEFAULT);
+ }
+
+ public void setExternalFieldPushdown(boolean externalFieldPushDown) {
+ setBoolean(EXTERNAL_FIELD_PUSHDOWN, externalFieldPushDown);
+ }
+
+ public boolean getSubplanMerge() {
+ return getBoolean(SUBPLAN_MERGE, AlgebricksConfig.SUBPLAN_MERGE_DEFAULT);
+ }
+
+ public void setSubplanMerge(boolean value) {
+ setBoolean(SUBPLAN_MERGE, value);
+ }
+
+ public boolean getSubplanNestedPushdown() {
+ return getBoolean(SUBPLAN_NESTEDPUSHDOWN, AlgebricksConfig.SUBPLAN_NESTEDPUSHDOWN_DEFAULT);
+ }
+
+ public void setSubplanNestedPushdown(boolean value) {
+ setBoolean(SUBPLAN_NESTEDPUSHDOWN, value);
+ }
+
+ public boolean getMinMemoryAllocation() {
+ return getBoolean(MIN_MEMORY_ALLOCATION, AlgebricksConfig.MIN_MEMORY_ALLOCATION_DEFAULT);
+ }
+
+ public void setMinMemoryAllocation(boolean value) {
+ setBoolean(MIN_MEMORY_ALLOCATION, value);
+ }
+
+ public int getExternalScanBufferSize() {
+ return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
+ }
+
+ public void setExternalScanBufferSize(int bufferSize) {
+ setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}
[asterixdb] 13/25: [NO ISSUE][OTH] Use IntOpenHashSet for datasets
ids
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9c01346c63551d1bae4bd8aa900365a8522f7b71
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Thu Apr 1 14:07:37 2021 +0300
[NO ISSUE][OTH] Use IntOpenHashSet for datasets ids
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Use IntOpenHashSet rather than Set<Integer> in StorageCleanupRequestMessage.
Change-Id: Ic7aea42f9a0fe9741d52ead7b6b13a85e2f65417
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10823
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../apache/asterix/app/message/StorageCleanupRequestMessage.java | 7 ++++---
.../apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java | 7 ++++---
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
index 85269a1..5e52517 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
@@ -21,7 +21,6 @@ package org.apache.asterix.app.message;
import static org.apache.hyracks.util.ExitUtil.EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED;
import java.util.Map;
-import java.util.Set;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
@@ -38,14 +37,16 @@ import org.apache.hyracks.util.ExitUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+
public class StorageCleanupRequestMessage extends CcIdentifiedMessage implements INcAddressedMessage {
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = LogManager.getLogger();
- private final Set<Integer> validDatasetIds;
+ private final IntOpenHashSet validDatasetIds;
private final long reqId;
- public StorageCleanupRequestMessage(long reqId, Set<Integer> validDatasetIds) {
+ public StorageCleanupRequestMessage(long reqId, IntOpenHashSet validDatasetIds) {
this.validDatasetIds = validDatasetIds;
this.reqId = reqId;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index cf2af95..ae50880 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -21,7 +21,6 @@ package org.apache.asterix.hyracks.bootstrap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -56,6 +55,8 @@ import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+
public class GlobalRecoveryManager implements IGlobalRecoveryManager {
private static final Logger LOGGER = LogManager.getLogger();
@@ -134,11 +135,11 @@ public class GlobalRecoveryManager implements IGlobalRecoveryManager {
protected void performGlobalStorageCleanup(MetadataTransactionContext mdTxnCtx, int storageGlobalCleanupTimeoutSecs)
throws Exception {
List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
- Set<Integer> validDatasetIds = new HashSet<>();
+ IntOpenHashSet validDatasetIds = new IntOpenHashSet();
for (Dataverse dataverse : dataverses) {
List<Dataset> dataverseDatasets =
MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverse.getDataverseName());
- dataverseDatasets.stream().map(Dataset::getDatasetId).forEach(validDatasetIds::add);
+ dataverseDatasets.stream().mapToInt(Dataset::getDatasetId).forEach(validDatasetIds::add);
}
ICcApplicationContext ccAppCtx = (ICcApplicationContext) serviceCtx.getApplicationContext();
final List<String> ncs = new ArrayList<>(ccAppCtx.getClusterStateManager().getParticipantNodes());
[asterixdb] 11/25: [NO ISSUE][STO] Delete invalid indexes during
cluster global recovery
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 17670aab184fe12fa30dc79376e819e07fac43c4
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Mar 31 21:18:18 2021 +0300
[NO ISSUE][STO] Delete invalid indexes during cluster global recovery
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Before starting cluster global recovery, send to all NCs valid
dataset ids from the metadata node.
- Delete any invalid indexes on NCs based on the metadata received
from the CC.
- Add storage options to enable/disable global storage recovery.
This allows tests that create storage objects without using the
metadata node to bypass global cleanup.
- Add storage option to specify the timeout for nodes to perform
global storage cleanup.
- Add test case for global storage recovery.
- Adapt existing test cases that require bypassing global cleanup.
Change-Id: Idee73e57fa5879c3b9aab5f881bf848e225f874b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10784
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../app/message/StorageCleanupRequestMessage.java | 108 +++++++++++++++++++++
.../apache/asterix/app/message/VoidResponse.java | 63 ++++++++++++
.../hyracks/bootstrap/GlobalRecoveryManager.java | 30 ++++++
.../test/dataflow/GlobalStorageCleanupTest.java | 68 +++++++++++++
.../test/dataflow/LSMFlushRecoveryTest.java | 12 ++-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less.1.regexadm | 1 +
.../asterix/common/config/StorageProperties.java | 25 ++++-
9 files changed, 304 insertions(+), 5 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
new file mode 100644
index 0000000..85269a1
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import static org.apache.hyracks.util.ExitUtil.EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.dataflow.DatasetLocalResource;
+import org.apache.asterix.common.messaging.CcIdentifiedMessage;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
+import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.LocalResource;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class StorageCleanupRequestMessage extends CcIdentifiedMessage implements INcAddressedMessage {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER = LogManager.getLogger();
+ private final Set<Integer> validDatasetIds;
+ private final long reqId;
+
+ public StorageCleanupRequestMessage(long reqId, Set<Integer> validDatasetIds) {
+ this.validDatasetIds = validDatasetIds;
+ this.reqId = reqId;
+ }
+
+ @Override
+ public void handle(INcApplicationContext appContext) throws HyracksDataException, InterruptedException {
+ INCMessageBroker broker = (INCMessageBroker) appContext.getServiceContext().getMessageBroker();
+ PersistentLocalResourceRepository localResourceRepository =
+ (PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
+ Map<Long, LocalResource> localResources = localResourceRepository.loadAndGetAllResources();
+ for (LocalResource resource : localResources.values()) {
+ DatasetLocalResource lr = (DatasetLocalResource) resource.getResource();
+ if (MetadataIndexImmutableProperties.isMetadataDataset(lr.getDatasetId())) {
+ // skip metadata indexes
+ continue;
+ }
+ if (!validDatasetIds.contains(lr.getDatasetId())) {
+ LOGGER.warn("found invalid index {} with dataset id {}", resource.getPath(), lr.getDatasetId());
+ deleteInvalidIndex(appContext, localResourceRepository, resource);
+ }
+ }
+ try {
+ broker.sendMessageToPrimaryCC(new VoidResponse(reqId, null));
+ } catch (Exception e) {
+ LOGGER.error("failed to notify CC of storage clean up; halting...", e);
+ ExitUtil.halt(EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED);
+ }
+ }
+
+ private void deleteInvalidIndex(INcApplicationContext appContext,
+ PersistentLocalResourceRepository localResourceRepository, LocalResource resource)
+ throws HyracksDataException {
+ IDatasetLifecycleManager lcManager = appContext.getDatasetLifecycleManager();
+ String resourceRelPath = resource.getPath();
+ synchronized (lcManager) {
+ IIndex index;
+ index = lcManager.get(resourceRelPath);
+ if (index != null) {
+ LOGGER.warn("unregistering invalid index {}", resourceRelPath);
+ lcManager.unregister(resourceRelPath);
+ } else {
+ LOGGER.warn("initializing unregistered invalid index {}", resourceRelPath);
+ try {
+ index = resource.getResource().createInstance(appContext.getServiceContext());
+ } catch (Exception e) {
+ LOGGER.warn("failed to initialize invalid index {}", resourceRelPath, e);
+ }
+ }
+ localResourceRepository.delete(resourceRelPath);
+ if (index != null) {
+ index.destroy();
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return StorageCleanupRequestMessage.class.getSimpleName();
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java
new file mode 100644
index 0000000..6a51c2d
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/VoidResponse.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker.ResponseState;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.common.messaging.api.INcResponse;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * A response to a request only indicating success or failure
+ */
+public class VoidResponse implements ICcAddressedMessage, INcResponse {
+
+ private static final long serialVersionUID = 1L;
+ private final Long reqId;
+ private final Throwable failure;
+
+ public VoidResponse(Long reqId, Throwable failure) {
+ this.reqId = reqId;
+ this.failure = failure;
+ }
+
+ @Override
+ public void handle(ICcApplicationContext appCtx) throws HyracksDataException, InterruptedException {
+ ICCMessageBroker broker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+ broker.respond(reqId, this);
+ }
+
+ @Override
+ public void setResult(MutablePair<ResponseState, Object> result) {
+ if (failure != null) {
+ result.setLeft(ResponseState.FAILURE);
+ result.setRight(failure);
+ } else {
+ result.setLeft(ResponseState.SUCCESS);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "{ \"response\" : \"" + (failure == null ? "success" : failure.getClass().getSimpleName()) + "\"}";
+ }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index 8165316..cf2af95 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -18,11 +18,15 @@
*/
package org.apache.asterix.hyracks.bootstrap;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.asterix.app.message.StorageCleanupRequestMessage;
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
@@ -32,6 +36,7 @@ import org.apache.asterix.common.config.DatasetConfig.TransactionState;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.messaging.CCMessageBroker;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -111,6 +116,10 @@ public class GlobalRecoveryManager implements IGlobalRecoveryManager {
LOGGER.info("Starting Global Recovery");
MetadataManager.INSTANCE.init();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ if (appCtx.getStorageProperties().isStorageGlobalCleanup()) {
+ int storageGlobalCleanupTimeout = appCtx.getStorageProperties().getStorageGlobalCleanupTimeout();
+ performGlobalStorageCleanup(mdTxnCtx, storageGlobalCleanupTimeout);
+ }
mdTxnCtx = doRecovery(appCtx, mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
recoveryCompleted = true;
@@ -122,6 +131,27 @@ public class GlobalRecoveryManager implements IGlobalRecoveryManager {
}
}
+ protected void performGlobalStorageCleanup(MetadataTransactionContext mdTxnCtx, int storageGlobalCleanupTimeoutSecs)
+ throws Exception {
+ List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
+ Set<Integer> validDatasetIds = new HashSet<>();
+ for (Dataverse dataverse : dataverses) {
+ List<Dataset> dataverseDatasets =
+ MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverse.getDataverseName());
+ dataverseDatasets.stream().map(Dataset::getDatasetId).forEach(validDatasetIds::add);
+ }
+ ICcApplicationContext ccAppCtx = (ICcApplicationContext) serviceCtx.getApplicationContext();
+ final List<String> ncs = new ArrayList<>(ccAppCtx.getClusterStateManager().getParticipantNodes());
+ CCMessageBroker messageBroker = (CCMessageBroker) ccAppCtx.getServiceContext().getMessageBroker();
+ long reqId = messageBroker.newRequestId();
+ List<StorageCleanupRequestMessage> requests = new ArrayList<>();
+ for (int i = 0; i < ncs.size(); i++) {
+ requests.add(new StorageCleanupRequestMessage(reqId, validDatasetIds));
+ }
+ messageBroker.sendSyncRequestToNCs(reqId, ncs, requests,
+ TimeUnit.SECONDS.toMillis(storageGlobalCleanupTimeoutSecs));
+ }
+
protected MetadataTransactionContext doRecovery(ICcApplicationContext appCtx, MetadataTransactionContext mdTxnCtx)
throws Exception {
// Loop over datasets
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java
new file mode 100644
index 0000000..84107fb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalStorageCleanupTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import java.io.File;
+
+import org.apache.asterix.app.bootstrap.TestNodeController;
+import org.apache.asterix.test.common.TestHelper;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class GlobalStorageCleanupTest {
+
+ public static final Logger LOGGER = LogManager.getLogger();
+ private static TestNodeController nc;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ System.out.println("SetUp: ");
+ TestHelper.deleteExistingInstanceFiles();
+ String configPath = System.getProperty("user.dir") + File.separator + "src" + File.separator + "test"
+ + File.separator + "resources" + File.separator + "cc.conf";
+ nc = new TestNodeController(configPath, false);
+ }
+
+ @Test
+ public void globalStorageCleanup() throws Exception {
+ nc.init(true);
+ LSMFlushRecoveryTest.nc = nc;
+ LSMFlushRecoveryTest lsmFlushRecoveryTest = new LSMFlushRecoveryTest();
+ lsmFlushRecoveryTest.initializeTestCtx();
+ lsmFlushRecoveryTest.createIndex();
+ lsmFlushRecoveryTest.readIndex();
+ nc.deInit(false);
+ nc.init(false);
+ // the index should deleted after the node initialization
+ lsmFlushRecoveryTest.initializeTestCtx();
+ boolean failedToReadIndex = false;
+ try {
+ lsmFlushRecoveryTest.readIndex();
+ } catch (Exception e) {
+ failedToReadIndex = true;
+ Assert.assertTrue(e.getMessage().contains(ErrorCode.INDEX_DOES_NOT_EXIST.errorCode()));
+ }
+ Assert.assertTrue(failedToReadIndex);
+ nc.deInit(false);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index c3a6839..9c6e95e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -77,7 +77,7 @@ import org.junit.Test;
public class LSMFlushRecoveryTest {
public static final Logger LOGGER = LogManager.getLogger();
- private static TestNodeController nc;
+ public static TestNodeController nc;
private static Dataset dataset;
private static PrimaryIndexInfo[] primaryIndexInfos;
private static SecondaryIndexInfo[] secondaryIndexInfo;
@@ -156,6 +156,10 @@ public class LSMFlushRecoveryTest {
}
private void initializeNc(boolean cleanUpOnStart) throws Exception {
+ // disable global clean up for this test to allow internal index creation
+ List<Pair<IOption, Object>> opts = new ArrayList<>();
+ opts.add(Pair.of(Option.STORAGE_GLOBAL_CLEANUP, false));
+ nc.setOpts(opts);
nc.init(cleanUpOnStart);
ncAppCtx = nc.getAppRuntimeContext();
// Override the LSMIOScheduler to avoid halting on failure and enable
@@ -177,7 +181,7 @@ public class LSMFlushRecoveryTest {
dsLifecycleMgr = ncAppCtx.getDatasetLifecycleManager();
}
- private void createIndex() throws Exception {
+ public void createIndex() throws Exception {
dataset = StorageTestUtils.DATASET;
secondaryIndexEntity = new Index(dataset.getDataverseName(), dataset.getDatasetName(), SECONDARY_INDEX_NAME,
SECONDARY_INDEX_TYPE, SECONDARY_INDEX_FIELD_NAMES, SECONDARY_INDEX_FIELD_INDICATORS,
@@ -193,7 +197,7 @@ public class LSMFlushRecoveryTest {
}
- private void initializeTestCtx() throws Exception {
+ public void initializeTestCtx() throws Exception {
JobId jobId = nc.newJobId();
testCtxs = new IHyracksTaskContext[NUM_PARTITIONS];
for (int i = 0; i < NUM_PARTITIONS; i++) {
@@ -203,7 +207,7 @@ public class LSMFlushRecoveryTest {
new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL));
}
- private void readIndex() throws HyracksDataException {
+ public void readIndex() throws HyracksDataException {
primaryIndexDataflowHelpers = new IIndexDataflowHelper[NUM_PARTITIONS];
primaryIndexes = new TestLsmBtree[NUM_PARTITIONS];
for (int i = 0; i < NUM_PARTITIONS; i++) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index eba50dc..f4dfdd1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -44,6 +44,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 98faa65..8fc48f9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -44,6 +44,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index a92f7d1..ed265e5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -44,6 +44,7 @@
"replication\.timeout" : 30,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
+ "storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index d9463bf..12c9c68 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE;
import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
@@ -27,6 +28,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@ -57,7 +59,9 @@ public class StorageProperties extends AbstractProperties {
STORAGE_WRITE_RATE_LIMIT(LONG_BYTE_UNIT, 0l),
STORAGE_MAX_CONCURRENT_FLUSHES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
STORAGE_MAX_SCHEDULED_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 8),
- STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2);
+ STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION(NONNEGATIVE_INTEGER, 2),
+ STORAGE_GLOBAL_CLEANUP(BOOLEAN, true),
+ STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int) TimeUnit.MINUTES.toSeconds(10));
private final IOptionType interpreter;
private final Object defaultValue;
@@ -72,6 +76,8 @@ public class StorageProperties extends AbstractProperties {
switch (this) {
case STORAGE_COMPRESSION_BLOCK:
case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
+ case STORAGE_GLOBAL_CLEANUP:
+ case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
return Section.COMMON;
default:
return Section.NC;
@@ -119,6 +125,10 @@ public class StorageProperties extends AbstractProperties {
return "The maximum number of scheduled merges per partition (0 means unlimited)";
case STORAGE_MAX_CONCURRENT_MERGES_PER_PARTITION:
return "The maximum number of concurrently executed merges per partition (0 means unlimited)";
+ case STORAGE_GLOBAL_CLEANUP:
+ return "Indicates whether or not global storage cleanup is performed";
+ case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
+ return "The maximum time to wait for nodes to respond to global storage cleanup requests";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -138,6 +148,11 @@ public class StorageProperties extends AbstractProperties {
public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) {
return null;
}
+
+ @Override
+ public boolean hidden() {
+ return this == STORAGE_GLOBAL_CLEANUP;
+ }
}
public static final long MAX_HEAP_BYTES = Runtime.getRuntime().maxMemory();
@@ -227,6 +242,14 @@ public class StorageProperties extends AbstractProperties {
return value != 0 ? value * numPartitions : Integer.MAX_VALUE;
}
+ public boolean isStorageGlobalCleanup() {
+ return accessor.getBoolean(Option.STORAGE_GLOBAL_CLEANUP);
+ }
+
+ public int getStorageGlobalCleanupTimeout() {
+ return accessor.getInt(Option.STORAGE_GLOBAL_CLEANUP_TIMEOUT);
+ }
+
protected int getMetadataDatasets() {
return MetadataIndexImmutableProperties.METADATA_DATASETS_COUNT;
}
[asterixdb] 23/25: [ASTERIXDB-2858][EXT]: Retry upon failure for S3
retryable errors
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5b5ac48f8829b63c906b4f28e6a959137be2b163
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Tue Apr 6 15:28:04 2021 +0300
[ASTERIXDB-2858][EXT]: Retry upon failure for S3 retryable errors
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Retry upon failure for S3 retryable errors.
Change-Id: Icec828b119fd959760281c4f4cc49449b779546e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10743
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Till Westmann
---
.../abstracts/AbstractExternalInputStream.java | 3 -
.../input/record/reader/aws/AwsS3InputStream.java | 74 +++++++++++++++++++---
.../record/reader/aws/AwsS3InputStreamFactory.java | 2 +-
.../external/util/ExternalDataConstants.java | 9 +++
.../asterix/external/util/ExternalDataUtils.java | 3 +-
.../input/record/reader/awss3/AwsS3Test.java | 72 +++++++++++++++++++++
6 files changed, 149 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
index 898f828..b37bce7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/abstracts/AbstractExternalInputStream.java
@@ -52,9 +52,6 @@ public abstract class AbstractExternalInputStream extends AbstractMultipleInputS
// Finished reading all the files
if (nextFileIndex >= filePaths.size()) {
- if (in != null) {
- CleanupUtils.close(in, null);
- }
return false;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 48035f3..4288cc4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -23,6 +23,7 @@ import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -31,18 +32,22 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.util.LogRedactionUtil;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
public class AwsS3InputStream extends AbstractExternalInputStream {
private final S3Client s3Client;
private final String bucket;
+ private static final int MAX_RETRIES = 5; // We will retry 5 times in case of internal error from AWS S3 service
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
@@ -58,24 +63,75 @@ public class AwsS3InputStream extends AbstractExternalInputStream {
// Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
// the header, then the S3 stream gets closed in the close method
- try {
- in = s3Client.getObject(getObjectRequest);
- } catch (NoSuchKeyException ex) {
- LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket "
- + getObjectRequest.bucket());
+ if (!doGetInputStream(getObjectRequest)) {
return false;
- } catch (SdkException ex) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
// Use gzip stream if needed
- String lowerCaseFileName = fileName.toLowerCase();
- if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
+ if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
}
return true;
}
+ /**
+ * Get the input stream. If an error is encountered, depending on the error code, a retry might be favorable.
+ *
+ * @return true
+ */
+ private boolean doGetInputStream(GetObjectRequest request) throws RuntimeDataException {
+ int retries = 0;
+ while (retries < MAX_RETRIES) {
+ try {
+ in = s3Client.getObject(request);
+ break;
+ } catch (NoSuchKeyException ex) {
+ LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(request.key()) + " was not found in bucket "
+ + request.bucket());
+ return false;
+ } catch (S3Exception ex) {
+ if (!shouldRetry(ex.awsErrorDetails().errorCode(), retries++)) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ LOGGER.debug(() -> "S3 retryable error: " + LogRedactionUtil.userData(ex.getMessage()));
+
+ // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ } catch (SdkException ex) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ }
+ return true;
+ }
+
+ private boolean shouldRetry(String errorCode, int currentRetry) {
+ return currentRetry < MAX_RETRIES && AwsS3.isRetryableError(errorCode);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (in != null) {
+ CleanupUtils.close(in, null);
+ }
+ if (s3Client != null) {
+ CleanupUtils.close(s3Client, null);
+ }
+ }
+
+ @Override
+ public boolean stop() {
+ try {
+ close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ return false;
+ }
+
private S3Client buildAwsS3Client(Map<String, String> configuration) throws HyracksDataException {
try {
return ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 747fcca..8197524 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -78,7 +78,7 @@ public class AwsS3InputStreamFactory extends AbstractExternalInputStreamFactory
// New API is not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode().equals(ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED)) {
filesOnly = oldApiListS3Objects(s3Client, container, includeExcludeMatcher);
} else {
throw ex;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 45d15df..047fe2a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -300,6 +300,15 @@ public class ExternalDataConstants {
public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
+
+ // AWS S3 specific error codes
+ public static final String ERROR_INTERNAL_ERROR = "InternalError";
+ public static final String ERROR_SLOW_DOWN = "SlowDown";
+ public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+
+ public static boolean isRetryableError(String errorCode) {
+ return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
+ }
}
public static class AzureBlob {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index d1bbe89..2d25d08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
@@ -785,7 +786,7 @@ public class ExternalDataUtils {
// Method not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
useOldApi = true;
response = isBucketEmpty(s3Client, container, prefix, true);
} else {
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
index 9cdeb16..90ea04b 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
@@ -18,16 +18,26 @@
*/
package org.apache.asterix.external.input.record.reader.awss3;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_SLOW_DOWN;
+
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStream;
import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStreamFactory;
+import org.apache.hyracks.api.exceptions.IFormattedException;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AwsS3Test {
@@ -70,4 +80,66 @@ public class AwsS3Test {
Assert.assertEquals(workload.getTotalSize(), 600);
}
}
+
+ @Test
+ public void s3InternalError() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with internal error code
+ AwsErrorDetails errorDetails = AwsErrorDetails.builder().errorCode(ERROR_INTERNAL_ERROR)
+ .errorMessage("Internal Error from AWS").build();
+ S3Exception internalErrorEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(internalErrorEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not internal error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. Internal Error from AWS"));
+ }
+ }
+
+ @Test
+ public void s3SlowDown() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with slow down error code
+ AwsErrorDetails errorDetails =
+ AwsErrorDetails.builder().errorCode(ERROR_SLOW_DOWN).errorMessage("SlowDown Error from AWS").build();
+ S3Exception slowDownEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(slowDownEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+
+ // Set S3Client
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not SlowDown error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. SlowDown Error from AWS"));
+ }
+ }
}
[asterixdb] 04/25: [NO ISSUE][COMP] Change default for synonym's
target object dataverse
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2e9815a4212f3fc39b8f5f1b00226b5339fefd3f
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Mar 26 19:14:49 2021 -0700
[NO ISSUE][COMP] Change default for synonym's target object dataverse
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- If target object dataverse is not specified in CREATE SYNONYM
statement then it should be set to the synonym's dataverse
Change-Id: I809900aa47a4d8ded2177e51b2ddba4a107f734b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10726
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../asterix/app/translator/QueryTranslator.java | 3 +-
.../synonym/synonym-03/synonym-03.1.ddl.sqlpp | 35 ++++++++++++++++++++++
.../synonym/synonym-03/synonym-03.2.query.sqlpp | 23 ++++++++++++++
.../results/synonym/synonym-03/synonym-03.2.adm | 2 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++++
.../src/main/markdown/sqlpp/7_ddl_dml.md | 3 +-
6 files changed, 69 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 64d0fd7..ab682c1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2639,7 +2639,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
CreateSynonymStatement css = (CreateSynonymStatement) stmt;
DataverseName dataverseName = getActiveDataverseName(css.getDataverseName());
String synonymName = css.getSynonymName();
- DataverseName objectDataverseName = getActiveDataverseName(css.getObjectDataverseName());
+ DataverseName objectDataverseName =
+ css.getObjectDataverseName() != null ? css.getObjectDataverseName() : dataverseName;
String objectName = css.getObjectName();
lockUtil.createSynonymBegin(lockManager, metadataProvider.getLocks(), dataverseName, synonymName);
try {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp
new file mode 100644
index 0000000..33397c9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.1.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that if the target object's dataverse is not specified then
+ * it defaults to the synonym's dataverse
+ */
+
+drop dataverse test1 if exists;
+create dataverse test1;
+
+drop dataverse test2 if exists;
+create dataverse test2;
+
+use test1;
+
+create synonym test2.md_syn_1 for `Metadata`.`Synonym`;
+
+create synonym test2.md_syn_2 for md_syn_1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp
new file mode 100644
index 0000000..8e0aaf9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/synonym/synonym-03/synonym-03.2.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select DataverseName, SynonymName, ObjectDataverseName, ObjectName
+from test2.md_syn_2
+where DataverseName like "test%"
+order by SynonymName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm
new file mode 100644
index 0000000..32200e3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/synonym/synonym-03/synonym-03.2.adm
@@ -0,0 +1,2 @@
+{ "DataverseName": "test2", "SynonymName": "md_syn_1", "ObjectDataverseName": "Metadata", "ObjectName": "Synonym" }
+{ "DataverseName": "test2", "SynonymName": "md_syn_2", "ObjectDataverseName": "test2", "ObjectName": "md_syn_1" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 7050d49..9dc621d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -10475,6 +10475,11 @@
<expected-error>ASX1063: Cannot find dataverse with name UNKNOWN_DATAVERSE</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="synonym">
+ <compilation-unit name="synonym-03">
+ <output-dir compare="Text">synonym-03</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="tokenizers">
<test-case FilePath="tokenizers">
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index e064f06..f32d603 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@ -437,8 +437,9 @@ The primary-key index can be identified by the fact that the `SearchKey` field i
The `CREATE SYNONYM` statement creates a synonym for a given dataset.
-This synonym may be used used instead of the dataset name in `SELECT`, `INSERT`, `UPSERT`, `DELETE`, and `LOAD` statements.
+This synonym may be used instead of the dataset name in `SELECT`, `INSERT`, `UPSERT`, `DELETE`, and `LOAD` statements.
The target dataset does not need to exist when the synonym is created.
+A synonym may be created for another synonym.
##### Example
[asterixdb] 16/25: [NO ISSUE][REP] Log details of files to be
replicated
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c3dbe4bbfbccdf24261112e95726986d434d5753
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Fri Apr 2 21:49:10 2021 +0300
[NO ISSUE][REP] Log details of files to be replicated
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Log details of the file to be replicated on master being sending it
to replicas.
- Log details of the file to be replicated before receiving it on a replica.
- Log details on file transfer failures.
Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../asterix/replication/management/NetworkingUtil.java | 16 +++++++++++++---
.../asterix/replication/messaging/ReplicateFileTask.java | 7 +++++++
.../asterix/replication/sync/FileSynchronizer.java | 4 ++++
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..97dd049 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@ import java.util.Enumeration;
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -72,9 +76,15 @@ public class NetworkingUtil {
long fileSize = fileChannel.size();
long count = fileSize;
long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
+ pos += numTransferred;
+ count -= numTransferred;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
+ count, numTransferred);
+ throw e;
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 35dbd49..5f7bee6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -63,6 +63,7 @@ public class ReplicateFileTask implements IReplicaTask {
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
try {
+ LOGGER.info("attempting to replicate {}", this);
final IIOManager ioManager = appCtx.getIoManager();
// resolve path
final FileReference localPath = ioManager.resolve(file);
@@ -127,4 +128,10 @@ public class ReplicateFileTask implements IReplicaTask {
final boolean isMetadata = input.readBoolean();
return new ReplicateFileTask(s, i, isMetadata);
}
+
+ @Override
+ public String toString() {
+ return "ReplicateFileTask{" + "file='" + file + '\'' + ", size=" + size + ", indexMetadata=" + indexMetadata
+ + '}';
+ }
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
index f0a4229..7bb2858 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
@@ -32,9 +32,12 @@ import org.apache.asterix.replication.messaging.ReplicationProtocol;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileSynchronizer {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
private final PartitionReplica replica;
@@ -53,6 +56,7 @@ public class FileSynchronizer {
final ISocketChannel channel = replica.getChannel();
final FileReference filePath = ioManager.resolve(file);
ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata);
+ LOGGER.info("attempting to replicate {} to replica {}", task, replica);
ReplicationProtocol.sendTo(replica, task);
// send the file itself
try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r");
[asterixdb] 09/25: [ASTERIXDB-2857][RT] Incorrect result for nested
loop outer join
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e48e9f2dc0c6f54082d463d47318fcade6b632c8
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Mar 30 17:12:27 2021 -0700
[ASTERIXDB-2857][RT] Incorrect result for nested loop outer join
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix incorrect number of unmatched tuples emitted
by nested loop implementation of left outer join
- Add RunFileWriter.eraseClosed() method
Change-Id: Ib56cf82fbe335a1d8f5d69caaf51e746db252202
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10785
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 ++++++++
.../leftouterjoin/query-ASTERIXDB-2857.plan | 39 +++++
.../query-ASTERIXDB-2857.1.ddl.sqlpp | 48 ++++++
.../query-ASTERIXDB-2857.2.update.sqlpp | 22 +++
.../query-ASTERIXDB-2857.3.query.sqlpp | 39 +++++
.../query-ASTERIXDB-2857.3.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../hyracks/dataflow/common/io/RunFileWriter.java | 10 +-
.../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 ++++++++++++---------
.../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +-
10 files changed, 326 insertions(+), 72 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
new file mode 100644
index 0000000..2304349
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
new file mode 100644
index 0000000..393f1db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
@@ -0,0 +1,39 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$136(ASC), $$137(ASC), $#3(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$136(ASC), $$137(ASC), $#3(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$127][$$128] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.tenk.tenk) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
new file mode 100644
index 0000000..87b5d75
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
new file mode 100644
index 0000000..2d7e768
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset tenk using localfs ((`path`=`asterix_nc1://data/tenk.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
new file mode 100644
index 0000000..823a540
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+USE test;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
new file mode 100644
index 0000000..1a31db8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
@@ -0,0 +1,2 @@
+{ "t0_unique1": 1891, "t1_unique1": 1891, "t2_unique1": 1891 }
+{ "t0_unique1": 8800, "t1_unique1": 8800, "t2_unique1": 8800 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 9dc621d..ce440c1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -12983,6 +12983,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="leftouterjoin">
+ <compilation-unit name="query-ASTERIXDB-2857">
+ <output-dir compare="Text">query-ASTERIXDB-2857</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
<compilation-unit name="right_branch_opt_1">
<output-dir compare="Text">right_branch_opt_1</output-dir>
</compilation-unit>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
index dabdd4f..c370b58 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
@@ -75,9 +75,15 @@ public class RunFileWriter implements IFrameWriter {
}
public void erase() throws HyracksDataException {
- close();
- file.delete();
+ try {
+ close();
+ } finally {
+ eraseClosed();
+ }
+ }
+ public void eraseClosed() {
+ file.delete();
// Make sure we never access the file if it is deleted.
file = null;
handle = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
index a5ad500..03ff72f 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
@@ -20,6 +20,7 @@ package org.apache.hyracks.dataflow.std.join;
import java.io.DataOutput;
import java.nio.ByteBuffer;
+import java.util.BitSet;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -28,6 +29,7 @@ import org.apache.hyracks.api.context.IHyracksJobletContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriter;
import org.apache.hyracks.api.dataflow.value.IPredicateEvaluator;
import org.apache.hyracks.api.dataflow.value.ITuplePairComparator;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -43,6 +45,14 @@ import org.apache.hyracks.dataflow.std.buffermanager.VariableFrameMemoryManager;
import org.apache.hyracks.dataflow.std.buffermanager.VariableFramePool;
public class NestedLoopJoin {
+ // Note: Min memory budget should be less than {@code AbstractJoinPOperator.MIN_FRAME_LIMIT_FOR_JOIN}
+ // Inner join: 1 frame for the outer input side, 1 frame for the inner input side, 1 frame for the output
+ private static final int MIN_FRAME_BUDGET_INNER_JOIN = 3;
+ // Outer join extra: Add 1 frame for the {@code outerMatchLOJ} bitset
+ private static final int MIN_FRAME_BUDGET_OUTER_JOIN = MIN_FRAME_BUDGET_INNER_JOIN + 1;
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ private static final int ESTIMATE_AVG_TUPLE_SIZE = 128;
+
private final FrameTupleAccessor accessorInner;
private final FrameTupleAccessor accessorOuter;
private final FrameTupleAppender appender;
@@ -54,30 +64,45 @@ public class NestedLoopJoin {
private final boolean isLeftOuter;
private final ArrayTupleBuilder missingTupleBuilder;
private final IPredicateEvaluator predEvaluator;
- private boolean isReversed; //Added for handling correct calling for predicate-evaluator upon recursive calls (in OptimizedHybridHashJoin) that cause role-reversal
+ // Added for handling correct calling for predicate-evaluator upon recursive calls
+ // (in OptimizedHybridHashJoin) that cause role-reversal
+ private final boolean isReversed;
private final BufferInfo tempInfo = new BufferInfo(null, -1, -1);
+ private final BitSet outerMatchLOJ;
public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
- FrameTupleAccessor accessorInner, int memSize, IPredicateEvaluator predEval, boolean isLeftOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
IMissingWriter[] missingWriters) throws HyracksDataException {
+ this(jobletContext, accessorOuter, accessorInner, memBudgetInFrames, predEval, isLeftOuter, missingWriters,
+ false);
+ }
+
+ public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
+ IMissingWriter[] missingWriters, boolean isReversed) throws HyracksDataException {
this.accessorInner = accessorInner;
this.accessorOuter = accessorOuter;
this.appender = new FrameTupleAppender();
this.outBuffer = new VSizeFrame(jobletContext);
this.innerBuffer = new VSizeFrame(jobletContext);
this.appender.reset(outBuffer, true);
- if (memSize < 3) {
- throw new HyracksDataException("Not enough memory is available for Nested Loop Join");
+
+ int minMemBudgetInFrames = isLeftOuter ? MIN_FRAME_BUDGET_OUTER_JOIN : MIN_FRAME_BUDGET_INNER_JOIN;
+ if (memBudgetInFrames < minMemBudgetInFrames) {
+ throw new HyracksDataException(ErrorCode.INSUFFICIENT_MEMORY);
}
+ int outerBufferMngrMemBudgetInFrames = memBudgetInFrames - minMemBudgetInFrames + 1;
+ int outerBufferMngrMemBudgetInBytes = jobletContext.getInitialFrameSize() * outerBufferMngrMemBudgetInFrames;
this.outerBufferMngr = new VariableFrameMemoryManager(
- new VariableFramePool(jobletContext, jobletContext.getInitialFrameSize() * (memSize - 2)),
- FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, memSize - 2));
+ new VariableFramePool(jobletContext, outerBufferMngrMemBudgetInBytes), FrameFreeSlotPolicyFactory
+ .createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, outerBufferMngrMemBudgetInFrames));
this.predEvaluator = predEval;
- this.isReversed = false;
-
this.isLeftOuter = isLeftOuter;
if (isLeftOuter) {
+ if (isReversed) {
+ throw new HyracksDataException(ErrorCode.ILLEGAL_STATE, "Outer join cannot reverse roles");
+ }
int innerFieldCount = this.accessorInner.getFieldCount();
missingTupleBuilder = new ArrayTupleBuilder(innerFieldCount);
DataOutput out = missingTupleBuilder.getDataOutput();
@@ -85,9 +110,14 @@ public class NestedLoopJoin {
missingWriters[i].writeMissing(out);
missingTupleBuilder.addFieldEndOffset();
}
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ int outerMatchLOJCardinalityEstimate = outerBufferMngrMemBudgetInBytes / ESTIMATE_AVG_TUPLE_SIZE;
+ outerMatchLOJ = new BitSet(Math.max(outerMatchLOJCardinalityEstimate, 1));
} else {
missingTupleBuilder = null;
+ outerMatchLOJ = null;
}
+ this.isReversed = isReversed;
FileReference file =
jobletContext.createManagedWorkspaceFile(this.getClass().getSimpleName() + this.toString());
@@ -117,23 +147,7 @@ public class NestedLoopJoin {
return;
}
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
- RunFileReader runFileReader = runFileWriter.createReader();
- try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
- } finally {
- runFileReader.close();
- }
+ multiBlockJoin(writer);
outerBufferMngr.reset();
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
throw new HyracksDataException("The given outer frame of size:" + outerBuffer.capacity()
@@ -142,16 +156,51 @@ public class NestedLoopJoin {
}
}
- private void blockJoin(BufferInfo outerBufferInfo, ByteBuffer innerBuffer, IFrameWriter writer)
- throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- accessorInner.reset(innerBuffer);
- int tupleCount0 = accessorOuter.getTupleCount();
- int tupleCount1 = accessorInner.getTupleCount();
+ private void multiBlockJoin(IFrameWriter writer) throws HyracksDataException {
+ int outerBufferFrameCount = outerBufferMngr.getNumFrames();
+ if (outerBufferFrameCount == 0) {
+ return;
+ }
+ RunFileReader runFileReader = runFileWriter.createReader();
+ try {
+ runFileReader.open();
+ if (isLeftOuter) {
+ outerMatchLOJ.clear();
+ }
+ while (runFileReader.nextFrame(innerBuffer)) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerTupleCount = accessorOuter.getTupleCount();
+ accessorInner.reset(innerBuffer.getBuffer());
+ blockJoin(outerTupleRunningCount, writer);
+ outerTupleRunningCount += outerTupleCount;
+ }
+ }
+ if (isLeftOuter) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerFrameTupleCount = accessorOuter.getTupleCount();
+ appendMissing(outerTupleRunningCount, outerFrameTupleCount, writer);
+ outerTupleRunningCount += outerFrameTupleCount;
+ }
+ }
+ } finally {
+ runFileReader.close();
+ }
+ }
- for (int i = 0; i < tupleCount0; ++i) {
+ private void blockJoin(int outerTupleStartPos, IFrameWriter writer) throws HyracksDataException {
+ int outerTupleCount = accessorOuter.getTupleCount();
+ int innerTupleCount = accessorInner.getTupleCount();
+ for (int i = 0; i < outerTupleCount; ++i) {
boolean matchFound = false;
- for (int j = 0; j < tupleCount1; ++j) {
+ for (int j = 0; j < innerTupleCount; ++j) {
int c = tpComparator.compare(accessorOuter, i, accessorInner, j);
boolean prdEval = evaluatePredicate(i, j);
if (c == 0 && prdEval) {
@@ -159,13 +208,8 @@ public class NestedLoopJoin {
appendToResults(i, j, writer);
}
}
-
- if (!matchFound && isLeftOuter) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ if (isLeftOuter && matchFound) {
+ outerMatchLOJ.set(outerTupleStartPos + i);
}
}
}
@@ -191,15 +235,18 @@ public class NestedLoopJoin {
FrameUtils.appendConcatToWriter(writer, appender, accessor1, tupleId1, accessor2, tupleId2);
}
- private void appendMissing(BufferInfo outerBufferInfo, IFrameWriter writer) throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- int tupleCount = accessorOuter.getTupleCount();
- for (int i = 0; i < tupleCount; ++i) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ private void appendMissing(int outerFrameMngrStartPos, int outerFrameTupleCount, IFrameWriter writer)
+ throws HyracksDataException {
+ int limit = outerFrameMngrStartPos + outerFrameTupleCount;
+ for (int outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerFrameMngrStartPos); outerTuplePos < limit; outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerTuplePos + 1)) {
+ int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
+ byte[] ntByteArray = missingTupleBuilder.getByteArray();
+ int ntSize = missingTupleBuilder.getSize();
+ int outerAccessorTupleIndex = outerTuplePos - outerFrameMngrStartPos;
+ FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, outerAccessorTupleIndex, ntFieldEndOffsets,
+ ntByteArray, 0, ntSize);
}
}
@@ -210,22 +257,10 @@ public class NestedLoopJoin {
}
public void completeJoin(IFrameWriter writer) throws HyracksDataException {
- RunFileReader runFileReader = runFileWriter.createDeleteOnCloseReader();
try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
+ multiBlockJoin(writer);
} finally {
- runFileReader.close();
+ runFileWriter.eraseClosed();
}
appender.write(writer, true);
}
@@ -233,8 +268,4 @@ public class NestedLoopJoin {
public void releaseMemory() throws HyracksDataException {
outerBufferMngr.reset();
}
-
- public void setIsReversed(boolean b) {
- this.isReversed = b;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index c142113..bb79981 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -802,11 +802,10 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
// The nested loop join result is outer + inner. All the other operator is probe + build.
// Hence the reverse relation is different.
boolean isReversed = outerRd == buildRd && innerRd == probeRd;
- assert isLeftOuter ? !isReversed : true : "LeftOut Join can not reverse roles";
ITuplePairComparator nljComptorOuterInner = isReversed ? buildComp : probComp;
NestedLoopJoin nlj = new NestedLoopJoin(jobletCtx, new FrameTupleAccessor(outerRd),
- new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter);
- nlj.setIsReversed(isReversed);
+ new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter,
+ isReversed);
nlj.setComparator(nljComptorOuterInner);
IFrame cacheBuff = new VSizeFrame(jobletCtx);
[asterixdb] 12/25: Revert "[NO ISSUE][API] Decode request local
path"
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8555239369a0877eba5bf234315f2135ebed6818
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed Mar 31 08:42:42 2021 -0400
Revert "[NO ISSUE][API] Decode request local path"
Thie change loses encoded / characters in path; reverting
This reverts commit 2525d512f3cc7b3c3e5fdbc0486632b4fac51507.
Change-Id: Ic8684ed854baec1a7e1a5ca7a00d1ccdae0dd1dc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10803
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../java/org/apache/hyracks/http/server/AbstractServlet.java | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
index fa64003..514a7dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
@@ -22,8 +22,6 @@ import static com.fasterxml.jackson.databind.MapperFeature.SORT_PROPERTIES_ALPHA
import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.ConcurrentMap;
@@ -177,12 +175,7 @@ public abstract class AbstractServlet implements IServlet {
public String localPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
int queryStart = uri.indexOf('?');
- String localPath = queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
- try {
- return URLDecoder.decode(localPath, StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException e) {
- throw new IllegalArgumentException(e);
- }
+ return queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
}
public String servletPath(IServletRequest request) {
[asterixdb] 18/25: [NO ISSUE][*DB][MISC] Avoid appending trailing ;
on statements ending with a ;
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d4f3aba32bb71a8a370bcaea332a1e5c576af4b3
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Apr 4 11:33:34 2021 -0400
[NO ISSUE][*DB][MISC] Avoid appending trailing ; on statements ending with a ;
Change-Id: Iae6db76195ad834f01c8e35fc8a30d3b7148764c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10883
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
Contrib: Michael Blow <mb...@apache.org>
---
.../asterix/api/http/server/QueryServiceServlet.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 5153ebb..a043050 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -280,10 +280,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
buildResponseHeaders(requestRef, sessionOutput, param, responsePrinter, delivery);
responsePrinter.printHeaders();
- validateStatement(param.getStatement());
- String statementsText = param.getStatement() + ";";
+ String statement = param.getStatement();
+ statement = statement == null || (!statement.isEmpty() && statement.charAt(statement.length() - 1) == ';')
+ ? statement : (statement + ";");
+ validateStatement(statement);
if (param.isParseOnly()) {
- ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statementsText);
+ ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statement);
setAccessControlHeaders(request, response);
executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK);
response.setStatus(executionState.getHttpStatus());
@@ -296,9 +298,9 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
IStatementExecutor.StatementProperties statementProperties =
new IStatementExecutor.StatementProperties();
response.setStatus(HttpResponseStatus.OK);
- executeStatement(request, requestRef, statementsText, sessionOutput, resultProperties,
- statementProperties, stats, param, executionState, param.getOptionalParams(), statementParams,
- responsePrinter, warnings);
+ executeStatement(request, requestRef, statement, sessionOutput, resultProperties, statementProperties,
+ stats, param, executionState, param.getOptionalParams(), statementParams, responsePrinter,
+ warnings);
executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK);
}
errorCount = 0;