You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2021/04/07 12:48:06 UTC
[asterixdb] 25/25: Merge branch 'cheshire-cat' into 'master'
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 21cf9cbe1f228cbd7d161dbd0b23b55403253dd0
Merge: b0be3fc ce5f3aa
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Wed Apr 7 11:52:34 2021 +0300
Merge branch 'cheshire-cat' into 'master'
Change-Id: Ic9ebf963437bf2cac2fb48b43b41728f515133c6
.../org/apache/asterix/active/ActiveManager.java | 21 +-
.../org/apache/asterix/active/IActiveRuntime.java | 5 +
.../active/message/ActiveManagerMessage.java | 14 +-
.../active/message/ActiveStatsRequestMessage.java | 6 +-
.../active/message/StopRuntimeParameters.java | 12 +-
.../operators/physical/InvertedIndexPOperator.java | 4 +-
.../optimizer/rules/ConstantFoldingRule.java | 3 +-
.../rules/MetaFunctionToMetaVariableRule.java | 4 +-
.../rules/SetAsterixPhysicalOperatorsRule.java | 5 +-
.../rules/SweepIllegalNonfunctionalFunctions.java | 3 -
.../subplan/InlineAllNtsInSubplanVisitor.java | 84 +-
...InlineSubplanInputForNestedTupleSourceRule.java | 48 +-
.../rules/subplan/SubplanFlatteningUtil.java | 51 +-
.../rules/util/EquivalenceClassUtils.java | 84 +-
.../asterix/translator/AbstractLangTranslator.java | 21 +-
.../translator/LangExpressionToPlanTranslator.java | 22 +-
.../apache/asterix/api/common/APIFramework.java | 7 +-
.../api/http/server/ConnectorApiServlet.java | 7 +-
.../asterix/api/http/server/NCUdfApiServlet.java | 4 +-
.../api/http/server/QueryServiceServlet.java | 14 +-
.../api/http/server/RebalanceApiServlet.java | 5 +-
.../app/active/ActiveEntityEventsListener.java | 2 +-
.../asterix/app/function/DatasetRewriter.java | 6 +-
.../apache/asterix/app/function/FeedRewriter.java | 5 +-
.../app/message/StorageCleanupRequestMessage.java | 109 +
.../apache/asterix/app/message/VoidResponse.java | 63 +
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 22 +-
.../org/apache/asterix/app/nc/RecoveryManager.java | 19 +-
.../asterix/app/translator/QueryTranslator.java | 43 +-
.../hyracks/bootstrap/GlobalRecoveryManager.java | 31 +
.../test/dataflow/GlobalStorageCleanupTest.java | 68 +
.../test/dataflow/LSMFlushRecoveryTest.java | 12 +-
.../src/test/resources/TweetSent/roundtrip.py | 4 +
asterixdb/asterix-app/src/test/resources/cc.conf | 1 +
.../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 +
.../queries/subquery/in_let_8.sqlpp} | 43 +-
.../queries/subquery/query-ASTERIXDB-2845.sqlpp | 2 +-
.../optimizerts/results/ASTERIXDB-2402.plan | 28 +-
.../leftouterjoin/query-ASTERIXDB-2857.plan | 39 +
.../optimizerts/results/query-issue562_ps.plan | 58 +-
.../optimizerts/results/subquery/exists.plan | 2 +-
.../optimizerts/results/subquery/exists_ps.plan | 4 +-
.../optimizerts/results/subquery/in_let_3.plan | 4 +-
.../optimizerts/results/subquery/in_let_4.plan | 4 +-
.../optimizerts/results/subquery/in_let_5.plan | 4 +-
.../subquery/{in_let_3.plan => in_let_8.plan} | 60 +-
.../optimizerts/results/subquery/not_exists.plan | 2 +-
.../results/subquery/not_exists_ps.plan | 4 +-
.../results/subquery/query-ASTERIXDB-2845.plan | 145 +-
.../tpcds/query-ASTERIXDB-1581-correlated-2.plan | 399 +-
.../tpcds/query-ASTERIXDB-1581-correlated.plan | 407 +-
.../external_dataset.000.ddl.sqlpp | 66 +
.../external_dataset.001.query.sqlpp} | 7 +-
.../external_dataset.002.query.sqlpp} | 7 +-
.../external_dataset.003.query.sqlpp} | 7 +-
.../external_dataset.004.query.sqlpp} | 7 +-
.../external_dataset.005.query.sqlpp} | 7 +-
.../external_dataset.006.query.sqlpp} | 7 +-
.../external_dataset.999.ddl.sqlpp} | 6 +-
.../py_function_error.2.ddl.sqlpp | 5 +-
...2.ddl.sqlpp => py_function_error.5.query.sqlpp} | 12 +-
...2.ddl.sqlpp => py_function_error.6.query.sqlpp} | 12 +-
.../query-ASTERIXDB-2857.1.ddl.sqlpp} | 30 +-
.../query-ASTERIXDB-2857.2.update.sqlpp} | 6 +-
.../query-ASTERIXDB-2857.3.query.sqlpp} | 23 +-
.../in_let/in_let.8.query.sqlpp} | 21 +-
.../synonym-03/synonym-03.1.ddl.sqlpp} | 19 +-
.../synonym-03/synonym-03.2.query.sqlpp} | 9 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +
.../cluster_state_1_full.1.regexadm | 2 +
.../cluster_state_1_less.1.regexadm | 2 +
.../results/binary/query_id/query_id.1.adm | 6005 --------------------
.../custom-buffer-size/external_dataset.001.adm | 1 +
.../custom-buffer-size/external_dataset.002.adm | 1 +
.../custom-buffer-size/external_dataset.003.adm | 1 +
.../custom-buffer-size/external_dataset.004.adm | 25 +
.../custom-buffer-size/external_dataset.005.adm | 1 +
.../custom-buffer-size/external_dataset.006.adm | 25 +
.../py_function_error/py_function_error.3.json | 1 +
.../py_function_error/py_function_error.4.json | 1 +
.../query-ASTERIXDB-2857.3.adm | 2 +
.../special_chars_2/special_chars_2.2.adm | 6 +-
.../runtimets/results/subquery/in_let/in_let.8.adm | 2 +
.../results/synonym/synonym-03/synonym-03.2.adm | 2 +
.../resources/runtimets/testsuite_it_python.xml | 2 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 41 +-
.../asterix/common/api/IIdentifierMapper.java} | 11 +-
.../asterix/common/config/CompilerProperties.java | 17 +-
.../asterix/common/config/ExternalProperties.java | 8 +-
.../common/config/OptimizationConfUtil.java | 15 +
.../asterix/common/config/StorageProperties.java | 54 +-
.../common/config/TransactionProperties.java | 5 +-
.../asterix/common/exceptions/WarningUtil.java | 4 +-
.../asterix/common/metadata/DataverseName.java | 74 +-
.../common/utils/IdentifierMappingUtil.java} | 25 +-
.../asterix/common/utils/IdentifierUtil.java} | 18 +-
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../asterix/common/metadata/DataverseNameTest.java | 25 +-
.../asterix-doc/src/main/markdown/sqlpp/5_error.md | 2 +-
.../src/main/markdown/sqlpp/7_ddl_dml.md | 3 +-
.../apache/asterix/external/api/IRecordReader.java | 5 +
.../dataflow/AbstractFeedDataFlowController.java | 5 +
.../dataflow/FeedRecordDataFlowController.java | 6 +
.../external/dataset/adapter/FeedAdapter.java | 5 +
.../abstracts/AbstractExternalInputStream.java | 3 -
.../AbstractExternalInputStreamFactory.java | 32 +-
.../input/record/reader/aws/AwsS3InputStream.java | 82 +-
.../record/reader/aws/AwsS3InputStreamFactory.java | 30 +-
.../record/reader/azure/AzureBlobInputStream.java | 12 +-
.../reader/azure/AzureBlobInputStreamFactory.java | 30 +-
.../record/reader/stream/StreamRecordReader.java | 6 +-
.../input/stream/AsterixInputStreamReader.java | 11 +-
.../StandardUTF8ToModifiedUTF8DataOutput.java | 2 +-
.../external/library/PythonLibraryEvaluator.java | 17 +-
.../library/PythonLibraryEvaluatorFactory.java | 34 +-
.../external/util/ExternalDataConstants.java | 14 +-
.../asterix/external/util/ExternalDataUtils.java | 13 +-
.../input/record/reader/awss3/AwsS3Test.java | 145 +
.../org/apache/asterix/metadata/MetadataNode.java | 14 +-
.../metadata/declared/DatasetDataSource.java | 6 +
.../metadata/declared/MetadataManagerUtil.java | 8 +-
.../metadata/declared/MetadataProvider.java | 17 +-
.../apache/asterix/metadata/entities/Function.java | 7 +
.../asterix/metadata/feeds/FeedMetadataUtil.java | 9 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 6 +-
.../InvertedIndexResourceFactoryProvider.java | 5 +-
.../asterix/om/functions/BuiltinFunctions.java | 2 +-
.../replication/management/NetworkingUtil.java | 16 +-
.../replication/messaging/ReplicateFileTask.java | 7 +
.../asterix/replication/sync/FileSynchronizer.java | 4 +
.../functions/CreateQueryUIDDescriptor.java | 32 +-
.../service/transaction/TransactionManager.java | 12 +-
asterixdb/pom.xml | 2 +-
.../appended-resources/supplemental-models.xml | 32 +-
.../common/exceptions/AlgebricksException.java | 2 +
.../api/HeuristicCompilerFactoryBuilder.java | 3 +-
.../algebricks/algebricks-core/pom.xml | 2 +-
.../logical/RunningAggregateOperator.java | 5 +
.../algebra/util/OperatorManipulationUtil.java | 9 +-
.../algebricks/core/config/AlgebricksConfig.java | 3 +
.../algebricks/core/jobgen/impl/JobGenContext.java | 9 +-
.../rewriter/base/PhysicalOptimizationConfig.java | 9 +
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 +-
.../hyracks/api/exceptions/HyracksException.java | 2 +
.../api/exceptions/IFormattedException.java | 11 +
.../org/apache/hyracks/api/exceptions/Warning.java | 27 +-
.../org/apache/hyracks/api/util/InvokeUtil.java | 2 +-
.../control/common/controllers/NCConfig.java | 4 +
.../hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +-
.../hyracks/dataflow/common/io/RunFileWriter.java | 10 +-
.../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 +-
.../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +-
.../hyracks/http/server/AbstractServlet.java | 9 +-
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 2 +-
hyracks-fullstack/pom.xml | 4 +-
155 files changed, 2427 insertions(+), 7219 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index b83fbdf,919a24e..ccd38d3
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@@ -134,9 -135,10 +135,10 @@@ public class APIFramework
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY,
+ CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
- CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
+ CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ce6e10d,d8698f9..d2e9284
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -18,7 -18,8 +18,9 @@@
*/
package org.apache.asterix.app.translator;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
+import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
import java.io.File;
import java.io.FileInputStream;
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 8064c05,a58673f..5538a0a
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@@ -160,8 -165,8 +169,10 @@@ public class CompilerProperties extend
public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
+ public static final String COMPILER_ARRAYINDEX_KEY = Option.COMPILER_ARRAYINDEX.ini();
+
+ public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
+
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
public CompilerProperties(PropertiesAccessor accessor) {
@@@ -228,7 -233,7 +239,11 @@@
return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
}
+ public boolean isArrayIndex() {
+ return accessor.getBoolean(Option.COMPILER_ARRAYINDEX);
+ }
++
+ public int getExternalScanMemorySize() {
+ return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
+ }
}
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 81974fa,c4dbd85..d498d69
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@@ -72,9 -72,9 +72,12 @@@ public class OptimizationConfUtil
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
boolean minMemoryAllocation = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
+ boolean arrayIndex = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_ARRAYINDEX_KEY,
+ compilerProperties.isArrayIndex());
+ int externalScanBufferSize = getExternalScanBufferSize(
+ (String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
+ compilerProperties.getExternalScanMemorySize(), sourceLoc);
+
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
@@@ -90,7 -90,7 +93,8 @@@
physOptConf.setSubplanMerge(subplanMerge);
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
+ physOptConf.setArrayIndexEnabled(arrayIndex);
+ physOptConf.setExternalScanBufferSize(externalScanBufferSize);
return physOptConf;
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 09709d0,3ec8aec..c769960
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@@ -19,7 -19,7 +19,8 @@@
package org.apache.asterix.metadata;
+import static org.apache.asterix.common.exceptions.ErrorCode.FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED;
+ import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.rmi.RemoteException;
import java.util.ArrayList;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9241458,ec63825..497b15f
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@@ -618,10 -589,9 +620,10 @@@ public class MetadataProvider implement
Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), indexName);
if (secondaryIndex == null) {
- throw new AlgebricksException(
- "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName());
+ throw new AlgebricksException("Code generation error: no index " + indexName + " for " + dataset() + " "
+ + dataset.getDatasetName());
}
+ Index.ValueIndexDetails secondaryIndexDetails = (Index.ValueIndexDetails) secondaryIndex.getIndexDetails();
RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
getSplitProviderAndConstraints(dataset, secondaryIndex.getIndexName());
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 944ddf7,817db5a..20632f1
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@@ -33,5 -34,6 +34,7 @@@ public class AlgebricksConfig
public static final boolean SUBPLAN_MERGE_DEFAULT = true;
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
+ public static final boolean ARRAY_INDEX_DEFAULT = false;
+ public static final int EXTERNAL_SCAN_BUFFER_SIZE =
+ StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
}
diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index f18cad2,bfff925..62bf671
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@@ -45,7 -45,7 +45,8 @@@ public class PhysicalOptimizationConfi
private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
+ private static final String ARRAY_INDEX = "ARRAY_INDEX";
+ private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
private final Properties properties = new Properties();
@@@ -227,14 -227,14 +228,22 @@@
setBoolean(MIN_MEMORY_ALLOCATION, value);
}
+ public boolean isArrayIndexEnabled() {
+ return getBoolean(ARRAY_INDEX, AlgebricksConfig.ARRAY_INDEX_DEFAULT);
+ }
+
+ public void setArrayIndexEnabled(boolean arrayIndex) {
+ setBoolean(ARRAY_INDEX, arrayIndex);
+ }
+
+ public int getExternalScanBufferSize() {
+ return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
+ }
+
+ public void setExternalScanBufferSize(int bufferSize) {
+ setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
+ }
+
private void setInt(String property, int value) {
properties.setProperty(property, Integer.toString(value));
}