You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2021/05/26 23:43:06 UTC
[asterixdb] branch master updated (e75c8ef -> e7974ce)
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from e75c8ef [ASTERIXDB-2892][IDX] Fixing bug w/ range queries for array indexes.
new 40be6e7 [NO ISSUE][HYR] Increase thread name brevity
new 94b1306 [NO ISSUE][REP] Account for SSL socket data transfer behavior
new 33647b6 [NO ISSUE][HYR][MISC] Clear interrupted state on halt
new c70687f [ASTERIXDB-2874][EXT] S3 external datasets: add support to anonymous authentication
new ef5467b [NO ISSUE][HYR][MISC] Elimate hashtable for thread name save/restore
new 605b115 [NO ISSUE][EXT] Provide direct access to controller stat primitives
new 2323a4c [ASTERIXDB-2886][COMP] Fix RemoveRedundantVariablesRule
new d5eb016 [NO ISSUE][OTH] Add operator field to subplan json plan
new d5edad2 [NO ISSUE][MISC] Misc / minor diagnostic improvement
new 0d88d11 [NO ISSUE][API] Make the AbstractServlet look for /* wild card
new 0da78d6 [NO ISSUE][OTH] Add request parameter to allow for immediate execution
new 7dfb0b3 [NO ISSUE][MISC] Ensure safe serialVersionUIDs for anon. inners
new 902f63c [NO ISSUE][HYR][HTTP] Set thread name on interruptable stream handler
new 61aad74 [NO ISSUE][FUN] Remove decode_dataverse_display_name()
new 82f1931 [NO ISSUE][OTH] Add util method to get request http scheme
new 3856cb3 [NO ISSUE] Update third-party dependencies
new 2fe0e14 Merge tracking branch 'gerrit/stabilization-f69489'
new 6abbeea Merge branch 'gerrit/stabilization-5949a1cb71'
new 8c67f9c Merge branch 'gerrit/mad-hatter'
new 404e382 [ASTERIXDB-2883][COMP] Improve null handling in UDF calls
new d28f9ad [NO ISSUE][MISC] Avoid URI.getHost/Port where hostnames may contain underscore
new 30c4a78 [NO ISSUE][OTH] Only order nodes in node group for new datasets
new ec5195a [ASTERIXDB-2897][FUN] Fix unknown handling in PyUDF
new 91f91d2 [NO ISSUE][EXT] Refactor JSONDataParser
new 5d02b03 [NO ISSUE][*DB] Enable large parsing resources to be freed on memory pressure
new a0d1fb7 [ASTERIXDB-2896] Increase UDF argument buffer size
new ed56b86 [NO ISSUE][TX] Ensure lock manager computes valid indexes for hash table
new b69e30e [NO ISSUE][MISC] Fix typo on active state log, += HyracksThrowingSupplier
new 0e7e4bd [NO ISSUE][STO] Close datasets of flushed indexes after recovery
new 86da3f8 [ASTERIXDB-2884] Compiler error with nested UDF calls
new dec8426 [NO ISSUE][MISC] Preserve Error instances on retryUntilSuccessOrExhausted
new d5de9eb [NO ISSUE] Fixed storage unit util false positives + added tests
new 2782be4 Merge branch 'gerrit/mad-hatter'
new eed4941 [ASTERIXDB-2894] Update UDF docs
new 528ee18 [NO ISSUE][*DB][EXT] Fail with helpful error message on non-JSON object
new 3b6982c [NO ISSUE][MISC] Improve parser error reporting
new 3435172 [NO ISSUE][HYR][HTTP] http stream handler -> consumer / processor
new e7974ce Merge branch 'cheshire-cat'
The 38 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../active/message/ActiveManagerMessage.java | 2 +-
.../ExtractBatchableExternalFunctionCallsRule.java | 12 +-
...duceDynamicTypeCastForExternalFunctionRule.java | 13 +-
.../asterix/translator/IRequestParameters.java | 2 +
.../asterix/translator/IStatementExecutor.java | 6 +-
.../apache/asterix/api/common/APIFramework.java | 24 +-
.../api/http/server/AbstractNCUdfServlet.java | 2 +-
.../app/active/ActiveEntityEventsListener.java | 7 +-
.../asterix/app/function/DumpIndexDatasource.java | 14 +-
.../asterix/app/function/DumpIndexRewriter.java | 6 +-
.../message/ExecuteStatementRequestMessage.java | 20 +-
.../org/apache/asterix/app/nc/RecoveryManager.java | 20 +-
.../asterix/app/nc/task/RetrieveLibrariesTask.java | 7 +-
.../asterix/app/translator/QueryTranslator.java | 16 +-
.../asterix/app/translator/RequestParameters.java | 17 +
.../org/apache/asterix/utils/FeedOperations.java | 2 +-
.../asterix/app/external/ExternalUDFLibrarian.java | 3 +-
.../apache/asterix/test/common/TestExecutor.java | 9 +-
.../test/dataflow/LSMFlushRecoveryTest.java | 12 +-
.../s3/anonymous_no_auth/test.000.ddl.sqlpp} | 18 +-
.../s3/anonymous_no_auth/test.001.ddl.sqlpp} | 18 +-
.../s3/anonymous_no_auth/test.002.ddl.sqlpp} | 17 +-
.../test.099.ddl.sqlpp | 0
....query.sqlpp => type_validation.10.query.sqlpp} | 7 +-
...on.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} | 0
.../type_validation.2.ddl.sqlpp | 4 +-
.../type_validation.3.query.sqlpp | 3 +-
...3.query.sqlpp => type_validation.4.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.5.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.6.query.sqlpp} | 7 +-
...3.query.sqlpp => type_validation.7.query.sqlpp} | 3 +-
...3.query.sqlpp => type_validation.8.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.9.query.sqlpp} | 5 +-
...ntiment.0.ddl.sqlpp => toplevel_fn.0.ddl.sqlpp} | 0
...ntiment.1.lib.sqlpp => toplevel_fn.1.lib.sqlpp} | 0
...ntiment.2.ddl.sqlpp => toplevel_fn.2.ddl.sqlpp} | 0
...ent.3.query.sqlpp => toplevel_fn.3.query.sqlpp} | 0
...ent.3.query.sqlpp => toplevel_fn.4.query.sqlpp} | 7 +-
...ntiment.4.ddl.sqlpp => toplevel_fn.5.ddl.sqlpp} | 0
.../type_name.0.ddl.sqlpp} | 0
.../type_name.1.lib.sqlpp} | 0
.../type_name.2.ddl.sqlpp} | 6 +-
.../type_name.3.query.sqlpp} | 11 +-
.../type_name.4.ddl.sqlpp} | 0
.../query-ASTERIXDB-2886.1.ddl.sqlpp} | 25 +-
.../query-ASTERIXDB-2886.2.ddl.sqlpp} | 25 +-
.../query-ASTERIXDB-2886.3.query.sqlpp} | 14 +-
.../special_chars_2/special_chars_2.2.query.sqlpp | 1 -
.../type_validation.1.adm | 2 +-
.../type_validation.2.adm} | 0
.../type_validation.3.adm | 1 +
.../type_validation.4.adm | 1 +
.../type_validation.5.adm} | 0
.../type_validation.6.adm} | 0
.../type_validation.7.adm | 1 +
.../type_validation.8.adm | 1 +
.../{toplevel_fn.1.adm => toplevel_fn.3.adm} | 0
.../toplevel_fn/toplevel_fn.4.adm} | 0
.../external-library/type_name/type_name.3.adm | 1 +
.../push-limit-to-primary-scan-select.11.adm | 4 +-
.../push-limit-to-primary-scan.7.adm | 4 +-
.../query-ASTERIXDB-2886.3.adm | 4 +
.../special_chars_2/special_chars_2.2.adm | 12 +-
.../results/union/union_opt_1/union_opt_1.11.adm | 8 +-
...stsuite_external_dataset_azure_blob_storage.xml | 10 +-
.../runtimets/testsuite_external_dataset_s3.xml | 17 +-
...testsuite_external_dataset_s3_one_partition.xml | 20 +-
.../resources/runtimets/testsuite_it_sqlpp.xml | 5 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../common/api/IDatasetLifecycleManager.java | 9 +
.../common/context/DatasetLifecycleManager.java | 11 +
.../asterix/common/exceptions/ErrorCode.java | 4 +-
.../common/transactions/IRecoveryManager.java | 11 -
.../src/main/resources/asx_errormsg/en.properties | 2 +
.../src/main/user-defined_function/udf.md | 63 ++-
.../dataflow/AbstractFeedDataFlowController.java | 5 +
.../dataflow/FeedRecordDataFlowController.java | 5 +-
.../dataflow/FeedStreamDataFlowController.java | 1 -
.../reader/stream/QuotedLineRecordReader.java | 2 +-
.../reader/stream/SemiStructuredRecordReader.java | 2 +-
.../ExternalScalarJavaFunctionEvaluator.java | 45 ++-
.../ExternalScalarPythonFunctionEvaluator.java | 30 +-
.../ExternalAssignBatchRuntimeFactory.java | 19 +-
...DataParser.java => AbstractJsonDataParser.java} | 105 ++---
.../external/parser/AbstractNestedDataParser.java | 2 +-
.../external/parser/DelimitedDataParser.java | 2 +-
.../asterix/external/parser/JSONDataParser.java | 437 ++-------------------
.../asterix/external/parser/ParseException.java | 4 +
.../asterix/external/parser/TweetParser.java | 2 +-
.../{ObjectPool.java => AbstractObjectPool.java} | 28 +-
.../external/parser/jackson/IObjectPool.java | 13 +-
.../external/parser/jackson/ObjectPool.java | 34 +-
.../external/parser/jackson/ParserContext.java | 8 +-
.../{ObjectPool.java => SoftObjectPool.java} | 38 +-
.../asterix/external/util/ExternalDataUtils.java | 64 ++-
.../asterix/external/library/TypeNameFactory.java | 14 +-
.../asterix/external/library/TypeNameFunction.java | 30 +-
.../asterix/metadata/entities/NodeGroup.java | 18 +-
.../functions/ExternalFunctionCompilerUtil.java | 2 +-
.../metadata/functions/ExternalTypeComputer.java | 13 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../utils/SecondaryIndexOperationsHelper.java | 4 +
.../asterix/om/functions/BuiltinFunctions.java | 3 -
.../org/apache/asterix/om/types/BuiltinType.java | 60 +--
.../replication/management/NetworkingUtil.java | 34 +-
.../DecodeDataverseDisplayNameDescriptor.java | 88 -----
.../runtime/functions/FunctionCollection.java | 2 -
.../asterix/runtime/utils/ClusterStateManager.java | 2 -
asterixdb/asterix-server/pom.xml | 4 +-
.../service/locking/ResourceGroupTable.java | 2 +-
asterixdb/pom.xml | 4 +-
.../appended-resources/supplemental-models.xml | 8 +-
....com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
.../LogicalOperatorPrettyPrintVisitorJson.java | 9 +-
.../rules/RemoveRedundantVariablesRule.java | 94 +++--
.../api/rewriter/runtime/SuperActivity.java | 5 +
.../runtime/SuperActivityOperatorNodePushable.java | 7 +-
.../apache/hyracks/api/util/ErrorMessageUtil.java | 11 +-
.../hyracks/api/util/HyracksThrowingSupplier.java} | 8 +-
.../org/apache/hyracks/api/util/InvokeUtil.java | 3 +
.../java/org/apache/hyracks}/util/ParseUtil.java | 25 +-
.../src/main/resources/errormsg/en.properties | 2 +-
.../apache/hyracks/control/cc/job/JobManager.java | 4 +-
.../file/FieldCursorForDelimitedDataParser.java | 5 +-
.../hyracks/http/server/AbstractServlet.java | 23 +-
.../hyracks/http/server/HttpServerHandler.java | 8 +-
.../apache/hyracks/http/server/utils/HttpUtil.java | 31 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 6 +-
.../util/MaintainedThreadNameExecutorService.java | 12 +-
.../java/org/apache/hyracks/util/NetworkUtil.java | 22 ++
.../java/org/apache/hyracks/util/StorageUtil.java | 53 +--
.../org/apache/hyracks/util/ThrowingFunction.java | 19 +
.../apache/hyracks/util/string/UTF8StringUtil.java | 11 +-
.../hyracks/util/string/UTF8StringWriter.java | 3 +-
.../org/apache/hyracks/util/NetworkUtilTest.java | 11 +
.../org/apache/hyracks/util/StorageUnitTest.java | 79 ++++
hyracks-fullstack/pom.xml | 18 +-
137 files changed, 1089 insertions(+), 1092 deletions(-)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.3.query.sqlpp => external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp} (66%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.3.query.sqlpp => external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp} (66%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.3.query.sqlpp => external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp} (70%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/{create-with-session-token => anonymous_no_auth}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.10.query.sqlpp} (83%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.4.query.sqlpp} (92%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.5.query.sqlpp} (87%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.6.query.sqlpp} (83%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.7.query.sqlpp} (96%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.8.query.sqlpp} (92%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/{type_validation.3.query.sqlpp => type_validation.9.query.sqlpp} (86%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.0.ddl.sqlpp => toplevel_fn.0.ddl.sqlpp} (100%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.1.lib.sqlpp => toplevel_fn.1.lib.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.2.ddl.sqlpp => toplevel_fn.2.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.3.query.sqlpp => toplevel_fn.3.query.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.3.query.sqlpp => toplevel_fn.4.query.sqlpp} (93%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/{mysentiment.4.ddl.sqlpp => toplevel_fn.5.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{type_validation/type_validation.0.ddl.sqlpp => type_name/type_name.0.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{type_validation/type_validation.1.lib.sqlpp => type_name/type_name.1.lib.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{python_open_type_validation/type_validation.2.ddl.sqlpp => type_name/type_name.2.ddl.sqlpp} (76%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{python_open_type_validation/type_validation.2.ddl.sqlpp => type_name/type_name.3.query.sqlpp} (78%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{type_validation/type_validation.4.ddl.sqlpp => type_name/type_name.4.ddl.sqlpp} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.3.query.sqlpp => misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp} (68%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.3.query.sqlpp => misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp} (68%)
rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/toplevel_fn/mysentiment.2.ddl.sqlpp => misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/{misc/query-ASTERIXDB-819/query-ASTERIXDB-819.1.adm => external-library/python_open_type_validation/type_validation.2.adm} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
copy asterixdb/asterix-app/src/test/resources/runtimets/results/{warnings/min-max-incompatible-types/min-max-incompatible-types.2.adm => external-library/python_open_type_validation/type_validation.5.adm} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/{warnings/min-max-incompatible-types/min-max-incompatible-types.2.adm => external-library/python_open_type_validation/type_validation.6.adm} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
rename asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/{toplevel_fn.1.adm => toplevel_fn.3.adm} (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/{warnings/min-max-incompatible-types/min-max-incompatible-types.2.adm => external-library/toplevel_fn/toplevel_fn.4.adm} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
copy asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/{JSONDataParser.java => AbstractJsonDataParser.java} (82%)
copy asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/{ObjectPool.java => AbstractObjectPool.java} (75%)
copy hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java => asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java (76%)
copy asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/{ObjectPool.java => SoftObjectPool.java} (55%)
copy hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java => asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java (70%)
copy hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java => asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java (50%)
delete mode 100644 asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java
rename asterixdb/src/main/licenses/content/{raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt => raw.githubusercontent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt} (100%)
copy hyracks-fullstack/hyracks/{hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java => hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java} (81%)
rename {asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external => hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks}/util/ParseUtil.java (56%)
create mode 100644 hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java
[asterixdb] 32/38: [NO ISSUE][MISC] Preserve Error instances on
retryUntilSuccessOrExhausted
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit dec84265a802476cf31d25d374cfd6511e0016c7
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun May 16 09:24:27 2021 -0400
[NO ISSUE][MISC] Preserve Error instances on retryUntilSuccessOrExhausted
Prior to this change, InvokeUtil.retryUntilSuccessOrExhausted() would wrap instances of
java.lang.Error with HyracksDataException upon exhaustion of retry attempts. Errors are
typically handled differently than non-Errors, so preserve the Error when propagating
the failure to the caller.
Change-Id: Idfe1d443addaed342b0c0ed3a0a3835ad226dbe7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11483
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../src/main/java/org/apache/hyracks/api/util/InvokeUtil.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index 0b1c5a6..7d04cf2 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@ -280,6 +280,9 @@ public class InvokeUtil {
long delayMs = delay.calculate(attempt);
if (!policy.retry(th) || span.elapsed() || span.remaining(TimeUnit.MILLISECONDS) < delayMs) {
onFailure.attemptFailed(action, attempt, true, span, failure);
+ if (th instanceof Error) {
+ throw (Error) th;
+ }
throw HyracksDataException.create(failure);
} else {
onFailure.attemptFailed(action, attempt, false, span, failure);
[asterixdb] 19/38: Merge branch 'gerrit/mad-hatter'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8c67f9cb4a380d23cdb3d3c04a85ff00e31b439d
Merge: 82f1931 6abbeea
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun May 2 12:32:23 2021 -0400
Merge branch 'gerrit/mad-hatter'
Change-Id: I9fb18d2460cc83005380020c52a294c06c4b264c
asterixdb/asterix-server/pom.xml | 4 ++--
asterixdb/pom.xml | 4 ++--
.../main/appended-resources/supplemental-models.xml | 8 ++++----
...tent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
hyracks-fullstack/pom.xml | 18 +++++++++---------
5 files changed, 17 insertions(+), 17 deletions(-)
diff --cc asterixdb/pom.xml
index 6067ef9,43e0dfd..a930643
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -75,19 -75,14 +75,19 @@@
<itest.excludes>${global.itest.excludes}</itest.excludes>
<license.stage>compile</license.stage>
<resource.stage>process-classes</resource.stage>
+ <prepare-env.stage>none</prepare-env.stage>
+ <shim.stage>none</shim.stage>
+ <pytestlib.stage>none</pytestlib.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <algebricks.version>0.3.5-SNAPSHOT</algebricks.version>
- <hyracks.version>0.3.5-SNAPSHOT</hyracks.version>
+ <algebricks.version>0.3.6-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.3.6-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<awsjavasdk.version>2.10.83</awsjavasdk.version>
+ <azurejavasdk.version>12.6.0</azurejavasdk.version>
+ <parquet.version>1.8.2</parquet.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
[asterixdb] 34/38: [ASTERIXDB-2894] Update UDF docs
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit eed4941d2d8dfb3395ae17a36c349ecac12c9cce
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Thu Apr 29 12:46:44 2021 -0700
[ASTERIXDB-2894] Update UDF docs
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Update API examples to include type
- Include details about typing and execution model
Change-Id: Id9780d72960f9094c29f7f5766185782069fe7cf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11225
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../src/main/user-defined_function/udf.md | 63 +++++++++++++++++-----
1 file changed, 50 insertions(+), 13 deletions(-)
diff --git a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
index 7ca23bb..655113b 100644
--- a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
+++ b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
@@ -19,7 +19,7 @@
## <a name="introduction">Introduction</a>
-Apache AsterixDB supports three languages for writing user-defined functions (UDFs): SQL++, Java and Python
+Apache AsterixDB supports three languages for writing user-defined functions (UDFs): SQL++, Java, and Python
A user can encapsulate data processing logic into a UDF and invoke it
later repeatedly. For SQL++ functions, a user can refer to [SQL++ Functions](sqlpp/manual.html#Functions)
for their usages. This document will focus on UDFs in languages other than SQL++
@@ -27,8 +27,10 @@ for their usages. This document will focus on UDFs in languages other than SQL++
## <a name="authentication">Endpoints and Authentication</a>
-The UDF endpoint is not enabled by default until authentication has been configured properly. To enable it, we
-will need to set the path to the credential file and populate it with our username and password.
+The UDF API endpoint used to deploy functions is not enabled by default until authentication has been configured properly.
+Even if the endpoint is enabled, it is only accessible on the loopback interface on each NC to restrict access.
+
+To enable it, we need to set the path to the credential file and populate it with our username and password.
The credential file is a simple `/etc/passwd` style text file with usernames and corresponding `bcrypt` hashed and salted
passwords. You can populate this on your own if you would like, but the `asterixhelper` utility can write the entries as
@@ -50,9 +52,7 @@ Now,restart the cluster if it was already started to allow the Cluster Controlle
## <a name="installingUDF">Installing a Java UDF Library</a>
To install a UDF package to the cluster, we need to send a Multipart Form-data HTTP request to the `/admin/udf` endpoint
-of the CC at the normal API port (`19002` by default). The request should use HTTP Basic authentication. This means your
-credentials will *not* be obfuscated or encrypted *in any way*, so submit to this endpoint over localhost or a network
-where you know your traffic is safe from eavesdropping. Any suitable tool will do, but for the example here I will use
+of the CC at the normal API port (`19004` by default). Any suitable tool will do, but for the example here I will use
`curl` which is widely available.
For example, to install a library with the following criteria:
@@ -65,7 +65,7 @@ For example, to install a library with the following criteria:
we would execute
- curl -v -u admin:admin -X POST -F 'data=@./lib.zip' localhost:19002/admin/udf/udfs/testlib
+ curl -v -u admin:admin -X POST -F 'data=@./lib.zip' -F 'type=java' localhost:19004/admin/udf/udfs/testlib
Any response other than `200` indicates an error in deployment.
@@ -119,7 +119,7 @@ scikit-learn here (our method is obviously better!), but it's just included as a
Then, deploy it the same as the Java UDF was, with the library name `pylib` in `udfs` dataverse
- curl -v -u admin:admin -X POST -F 'data=@./lib.pyz' localhost:19002/admin/udf/udfs/pylib
+ curl -v -u admin:admin -X POST -F 'data=@./lib.pyz' -F 'type=python' localhost:19002/admin/udf/udfs/pylib
With the library deployed, we can define a function within it for use. For example, to expose the Python function
`sentiment` in the module `sentiment_mod` in the class `sent_model`, the `CREATE FUNCTION` would be as follows
@@ -131,14 +131,14 @@ With the library deployed, we can define a function within it for use. For examp
AS "sentiment_mod", "sent_model.sentiment" AT pylib;
By default, AsterixDB will treat all external functions as deterministic. It means the function must return the same
-result for the same input, irrespective of when or how many times the function is called on that input.
-This particular function behaves the same on each input, so it satisfies the deterministic property.
+result for the same input, irrespective of when or how many times the function is called on that input.
+This particular function behaves the same on each input, so it satisfies the deterministic property.
This enables better optimization of queries including this function.
-If a function is not deterministic then it should be declared as such by using `WITH` sub-clause:
+If a function is not deterministic then it should be declared as such by using a `WITH` sub-clause:
USE udfs;
- CREATE FUNCTION sentiment(a)
+ CREATE FUNCTION sentiment(text)
AS "sentiment_mod", "sent_model.sentiment" AT pylib
WITH { "deterministic": false }
@@ -155,6 +155,43 @@ With the function now defined, it can then be used as any other scalar SQL++ fun
SELECT t.msg as msg, sentiment(t.msg) as sentiment
FROM Tweets t;
+## <a name="pytpes">Python Type Mappings</a>
+
+Currently only a subset of AsterixDB types are supported in Python UDFs. The supported types are as follows:
+
+- Integer types (int8,16,32,64)
+- Floating point types (float, double)
+- String
+- Boolean
+- Arrays, Sets (cast to lists)
+- Objects (cast to dict)
+
+Unsupported types can be cast to these in SQL++ first in order to be passed to a Python UDF
+
+## <a name="execution">Execution Model For UDFs</a>
+
+AsterixDB queries are deployed across the cluster as Hyracks jobs. A Hyracks job has a lifecycle that can be simplified
+for the purposes of UDFs to
+ - A pre-run phase which allocates resources, `open`
+ - The time during which the job has data flowing through it, `nextFrame`
+ - Cleanup and shutdown in `close`.
+
+If a SQL++ function is defined as a member of a class in the library, the class will be instantiated
+during `open`. The class will exist in memory for the lifetime of the query. Therefore if your function needs to reference
+files or other data that would be costly to load per-call, making it a member variable that is initialized in the constructor
+of the object will greatly increase the performance of the SQL++ function.
+
+For each function invoked during a query, there will be an independent instance of the function per data partition. This
+means that the function must not assume there is any global state or that it can assume things about the layout
+of the data. The execution of the function will be parallel to the same degree as the level of data parallelism in the
+cluster.
+
+After initialization, the function bound in the SQL++ function definition is called once per tuple during the query
+execution (i.e. `nextFrame`). Unless the function specifies `null-call` in the `WITH` clause, `NULL` values will be
+skipped.
+
+At the close of the query, the function is torn down and not re-used in any way. All functions should assume that
+nothing will persist in-memory outside of the lifetime of a query, and any behavior contrary to this is undefined.
## <a id="UDFOnFeeds">Attaching a UDF on Data Feeds</a>
@@ -239,7 +276,7 @@ If you want to uninstall the UDF library, simply issue a `DELETE` against the en
functions declared with the library are removed. First we'll drop the function we declared earlier:
USE udfs;
- DROP FUNCTION mysum@2;
+ DROP FUNCTION mysum(a,b);
Then issue the proper `DELETE` request
[asterixdb] 07/38: [ASTERIXDB-2886][COMP] Fix
RemoveRedundantVariablesRule
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2323a4cd9b8a9c278797aefdff61552836d32a9c
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Apr 20 14:38:53 2021 -0700
[ASTERIXDB-2886][COMP] Fix RemoveRedundantVariablesRule
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Change RemoveRedundantVariablesRule to
operate on the whole plan at once instead
of working incrementally on each operator
Change-Id: Ia6cec065614ad5e2b14e7b160cf3c9de7215618a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11103
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../query-ASTERIXDB-2886.1.ddl.sqlpp | 43 ++++++++++
.../query-ASTERIXDB-2886.2.ddl.sqlpp | 43 ++++++++++
.../query-ASTERIXDB-2886.3.query.sqlpp | 30 +++++++
.../push-limit-to-primary-scan-select.11.adm | 4 +-
.../push-limit-to-primary-scan.7.adm | 4 +-
.../query-ASTERIXDB-2886.3.adm | 4 +
.../results/union/union_opt_1/union_opt_1.11.adm | 8 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../rules/RemoveRedundantVariablesRule.java | 94 ++++++++++++----------
9 files changed, 186 insertions(+), 49 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
new file mode 100644
index 0000000..ab6b723
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test1 if exists;
+create dataverse test1;
+use test1;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
new file mode 100644
index 0000000..621d057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test2 if exists;
+create dataverse test2;
+use test2;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
new file mode 100644
index 0000000..e48a1e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select ds.DataverseName, ds.DatasetName, dt.Derived.IsAnonymous, indexes
+from Metadata.`Dataset` as ds left join Metadata.`Datatype` dt
+on ds.DataverseName = dt.DataverseName and ds.DatatypeName = dt.DatatypeName
+let indexes = (
+ select idx.IndexName
+ from Metadata.`Index` as idx
+ where idx.DataverseName = ds.DataverseName and idx.DatasetName = ds.DatasetName
+ order by idx.IndexName
+)
+where ds.DataverseName like "test%"
+order by ds.DataverseName, ds.DatasetName;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
index 85cf5c5..28c74ac 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan-select/push-limit-to-primary-scan-select.11.adm
@@ -2,9 +2,9 @@ distribute result [$$202]
-- DISTRIBUTE_RESULT |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |LOCAL|
- aggregate [$$202] <- [agg-sql-sum($$235)]
+ aggregate [$$202] <- [agg-sql-sum($$231)]
-- AGGREGATE |LOCAL|
- aggregate [$$235] <- [agg-sql-count(1)]
+ aggregate [$$231] <- [agg-sql-count(1)]
-- AGGREGATE |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
index f830e9b..f8a800a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/limit/push-limit-to-primary-scan/push-limit-to-primary-scan.7.adm
@@ -2,9 +2,9 @@ distribute result [$$180]
-- DISTRIBUTE_RESULT |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |LOCAL|
- aggregate [$$180] <- [agg-sql-sum($$209)]
+ aggregate [$$180] <- [agg-sql-sum($$205)]
-- AGGREGATE |LOCAL|
- aggregate [$$209] <- [agg-sql-count(1)]
+ aggregate [$$205] <- [agg-sql-count(1)]
-- AGGREGATE |LOCAL|
exchange
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000..7065d25
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
@@ -0,0 +1,4 @@
+{ "DataverseName": "test1", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test1", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
+{ "DataverseName": "test2", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test2", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
index 1b39645..a809a8e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/union/union_opt_1/union_opt_1.11.adm
@@ -6,7 +6,7 @@ distribute result [$$t]
-- STREAM_LIMIT |UNPARTITIONED|
exchange
-- RANDOM_MERGE_EXCHANGE |PARTITIONED|
- union ($$151, $$310, $$t)
+ union ($$151, $$178, $$t)
-- UNION_ALL |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -58,7 +58,7 @@ distribute result [$$t]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- union ($$345, $$356, $$310)
+ union ($$345, $$354, $$178)
-- UNION_ALL |PARTITIONED|
exchange
-- RANDOM_PARTITION_EXCHANGE |PARTITIONED|
@@ -84,9 +84,9 @@ distribute result [$$t]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
exchange
-- RANDOM_PARTITION_EXCHANGE |PARTITIONED|
- project ([$$356])
+ project ([$$354])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$356] <- [{"two": $$186}]
+ assign [$$354] <- [{"two": $$186}]
-- ASSIGN |PARTITIONED|
limit 4
-- STREAM_LIMIT |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 1f955e6..c9e37e4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6841,6 +6841,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2886">
+ <output-dir compare="Text">query-ASTERIXDB-2886</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="unsupported_parameter">
<output-dir compare="Text">none</output-dir>
<expected-error>Query parameter compiler.joinmem is not supported</expected-error>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
index 5386193..3760be5 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
@@ -36,7 +36,6 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -69,26 +68,33 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
private final VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor();
- private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap =
- new HashMap<LogicalVariable, List<LogicalVariable>>();
+
+ private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap = new HashMap<>();
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
if (context.checkIfInDontApplySet(this, opRef.getValue())) {
return false;
}
- boolean modified = removeRedundantVariables(opRef, context);
- if (modified) {
- context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
- }
- return modified;
+ clear();
+ return removeRedundantVariables(opRef, true, context);
+ }
+
+ private void clear() {
+ equivalentVarsMap.clear();
}
private void updateEquivalenceClassMap(LogicalVariable lhs, LogicalVariable rhs) {
List<LogicalVariable> equivalentVars = equivalentVarsMap.get(rhs);
if (equivalentVars == null) {
- equivalentVars = new ArrayList<LogicalVariable>();
+ equivalentVars = new ArrayList<>();
// The first element in the list is the bottom-most representative which will replace all equivalent vars.
equivalentVars.add(rhs);
equivalentVars.add(lhs);
@@ -97,12 +103,32 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
equivalentVarsMap.put(lhs, equivalentVars);
}
- private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
+ private LogicalVariable findEquivalentRepresentativeVar(LogicalVariable var) {
+ List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
+ if (equivalentVars == null) {
+ return null;
+ }
+ LogicalVariable representativeVar = equivalentVars.get(0);
+ return var.equals(representativeVar) ? null : representativeVar;
+ }
+
+ private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, boolean first,
+ IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (!first) {
+ context.addToDontApplySet(this, op);
+ }
+
LogicalOperatorTag opTag = op.getOperatorTag();
boolean modified = false;
+ // Recurse into children.
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ if (removeRedundantVariables(inputOpRef, false, context)) {
+ modified = true;
+ }
+ }
+
// Update equivalence class map.
if (opTag == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) op;
@@ -142,7 +168,7 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
- if (removeRedundantVariables(rootRef, context)) {
+ if (removeRedundantVariables(rootRef, false, context)) {
modified = true;
}
}
@@ -158,14 +184,8 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
if (modified) {
context.computeAndSetTypeEnvironmentForOperator(op);
- context.addToDontApplySet(this, op);
}
- // Clears the equivalent variable map if the current operator is the root operator
- // in the query plan.
- if (opTag == LogicalOperatorTag.DISTRIBUTE_RESULT || opTag == LogicalOperatorTag.SINK) {
- equivalentVarsMap.clear();
- }
return modified;
}
@@ -227,38 +247,34 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
* We cannot use the VariableSubstitutionVisitor here because the project ops
* maintain their variables as a list and not as expressions.
*/
- private boolean replaceProjectVars(ProjectOperator op) throws AlgebricksException {
+ private boolean replaceProjectVars(ProjectOperator op) {
List<LogicalVariable> vars = op.getVariables();
int size = vars.size();
boolean modified = false;
for (int i = 0; i < size; i++) {
LogicalVariable var = vars.get(i);
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- continue;
- }
- // Replace with equivalence class representative.
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
- vars.set(i, equivalentVars.get(0));
+ LogicalVariable representativeVar = findEquivalentRepresentativeVar(var);
+ if (representativeVar != null) {
+ // Replace with equivalence class representative.
+ vars.set(i, representativeVar);
modified = true;
}
}
return modified;
}
- private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksException {
+ private boolean replaceUnionAllVars(UnionAllOperator op) {
boolean modified = false;
for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : op.getVariableMappings()) {
- List<LogicalVariable> firstEquivalentVars = equivalentVarsMap.get(varMapping.first);
- List<LogicalVariable> secondEquivalentVars = equivalentVarsMap.get(varMapping.second);
// Replace variables with their representative.
- if (firstEquivalentVars != null) {
- varMapping.first = firstEquivalentVars.get(0);
+ LogicalVariable firstRepresentativeVar = findEquivalentRepresentativeVar(varMapping.first);
+ if (firstRepresentativeVar != null) {
+ varMapping.first = firstRepresentativeVar;
modified = true;
}
- if (secondEquivalentVars != null) {
- varMapping.second = secondEquivalentVars.get(0);
+ LogicalVariable secondRepresentativeVar = findEquivalentRepresentativeVar(varMapping.second);
+ if (secondRepresentativeVar != null) {
+ varMapping.second = secondRepresentativeVar;
modified = true;
}
}
@@ -269,17 +285,13 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
@Override
public boolean transform(Mutable<ILogicalExpression> exprRef) {
ILogicalExpression e = exprRef.getValue();
- switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+ switch (e.getExpressionTag()) {
case VARIABLE: {
// Replace variable references with their equivalent representative in the equivalence class map.
VariableReferenceExpression varRefExpr = (VariableReferenceExpression) e;
LogicalVariable var = varRefExpr.getVariableReference();
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- return false;
- }
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
+ LogicalVariable representative = findEquivalentRepresentativeVar(var);
+ if (representative != null) {
varRefExpr.setVariable(representative);
return true;
}
[asterixdb] 01/38: [NO ISSUE][HYR] Increase thread name brevity
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 40be6e72254ea9a2192041eb357435bc838dee93
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Apr 13 17:28:31 2021 -0400
[NO ISSUE][HYR] Increase thread name brevity
Change-Id: I55a7586980cf869edd383ab435952f10b3976b86
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11045
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../org/apache/hyracks/api/rewriter/runtime/SuperActivity.java | 5 +++++
.../api/rewriter/runtime/SuperActivityOperatorNodePushable.java | 7 +++----
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
index ddcfd78..55eaf68 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivity.java
@@ -54,6 +54,11 @@ public class SuperActivity extends OneToOneConnectedActivityCluster implements I
}
@Override
+ public String getDisplayName() {
+ return "SA";
+ }
+
+ @Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index 79687ab..e43d72a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -55,7 +55,7 @@ import org.apache.hyracks.api.util.ExceptionUtils;
* connected activities in a single thread.
*/
public class SuperActivityOperatorNodePushable implements IOperatorNodePushable {
- private static final String CLASS_ABBREVIATION = "SA";
+ private static final String CLASS_ABBREVIATION = "SAO";
private final Map<ActivityId, IOperatorNodePushable> operatorNodePushables = new HashMap<>();
private final List<IOperatorNodePushable> operatorNodePushablesBFSOrder = new ArrayList<>();
private final Map<ActivityId, IActivity> startActivities;
@@ -222,9 +222,8 @@ public class SuperActivityOperatorNodePushable implements IOperatorNodePushable
tasks.add(ctx.getExecutorService().submit(() -> {
startSemaphore.release();
try {
- Thread.currentThread()
- .setName(Thread.currentThread().getName() + ":" + ctx.getJobletContext().getJobId()
- + ":" + ctx.getTaskAttemptId() + ":" + CLASS_ABBREVIATION);
+ Thread.currentThread().setName(CLASS_ABBREVIATION + ":" + ctx.getJobletContext().getJobId()
+ + ":" + ctx.getTaskAttemptId());
action.run(op);
} catch (Throwable th) { // NOSONAR: Must catch all causes of failure
failures.offer(th);
[asterixdb] 12/38: [NO ISSUE][MISC] Ensure safe serialVersionUIDs
for anon. inners
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7dfb0b3625c4813e26ae6c4b4eb5b48823d6afc6
Author: Michael Blow <mb...@apache.org>
AuthorDate: Fri Apr 23 08:26:56 2021 -0400
[NO ISSUE][MISC] Ensure safe serialVersionUIDs for anon. inners
- user model changes: no
- storage format changes: no
- interface changes: yes
Ensure anonymous inner classes have unique & non-trivial serialVersionUIDs
when multiples exist of a given type in an enclosing class, to avoid
making hard-to-detect binary-incompatible changes
Change-Id: I073c8a820f76bea9e73064a9bca350c63f65a698
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11163
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../org/apache/asterix/om/types/BuiltinType.java | 60 +++++++++++-----------
.../apache/hyracks/control/cc/job/JobManager.java | 4 +-
2 files changed, 32 insertions(+), 32 deletions(-)
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
index 76ba6a2..bdac9e9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
@@ -45,7 +45,7 @@ public abstract class BuiltinType implements IAType {
/** the type of all types */
public static final BuiltinType ALL_TYPE = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -6449893468153263063L;
@Override
public ATypeTag getTypeTag() {
@@ -83,7 +83,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AINT8 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -3548486253352636441L;
@Override
public String getDisplayName() {
@@ -116,7 +116,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AINT16 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7372637344359144146L;
@Override
public String getDisplayName() {
@@ -149,7 +149,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AINT32 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -2019098705812691759L;
@Override
public String getDisplayName() {
@@ -182,7 +182,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AINT64 = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1512324312146380842L;
@Override
public ATypeTag getTypeTag() {
@@ -215,7 +215,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ABINARY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 6267661034873713699L;
@Override
public String getDisplayName() {
@@ -248,7 +248,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AFLOAT = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 3677347396583352706L;
@Override
public String getDisplayName() {
@@ -281,7 +281,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ADOUBLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -4541634684505456336L;
@Override
public String getDisplayName() {
@@ -314,7 +314,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ASTRING = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -7772076532516609205L;
@Override
public String getDisplayName() {
@@ -347,7 +347,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AMISSING = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7776297133205477953L;
@Override
public String getDisplayName() {
@@ -380,7 +380,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ANULL = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8499117420923359476L;
@Override
public String getDisplayName() {
@@ -413,7 +413,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ABOOLEAN = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -3974299905945243639L;
@Override
public String getDisplayName() {
@@ -446,7 +446,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ATIME = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 4868038022671454583L;
@Override
public String getDisplayName() {
@@ -479,7 +479,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ADATE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 4037452728204123123L;
@Override
public String getDisplayName() {
@@ -512,7 +512,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ADATETIME = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 8133030540756758494L;
@Override
public String getDisplayName() {
@@ -545,7 +545,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ADURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2724423960287103412L;
@Override
public String getDisplayName() {
@@ -578,7 +578,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AYEARMONTHDURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -5424462543021800169L;
@Override
public String getDisplayName() {
@@ -611,7 +611,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ADAYTIMEDURATION = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1605907151536226966L;
@Override
public String getDisplayName() {
@@ -644,7 +644,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AINTERVAL = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 6503802193278692294L;
@Override
public String getDisplayName() {
@@ -674,7 +674,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType APOINT = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8095235249315628525L;
@Override
public ATypeTag getTypeTag() {
@@ -707,7 +707,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType APOINT3D = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 8511348838989487264L;
@Override
public ATypeTag getTypeTag() {
@@ -740,7 +740,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ALINE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -5081002479268381298L;
@Override
public String getDisplayName() {
@@ -773,7 +773,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType APOLYGON = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1150256493732890410L;
@Override
public String getDisplayName() {
@@ -806,7 +806,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType AGEOMETRY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1561894254022047509L;
@Override
public String getDisplayName() {
@@ -838,7 +838,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ACIRCLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2609452848446856772L;
@Override
public String getDisplayName() {
@@ -871,7 +871,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ARECTANGLE = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 3751474563396177041L;
@Override
public String getDisplayName() {
@@ -904,7 +904,7 @@ public abstract class BuiltinType implements IAType {
public static final IAType ABITARRAY = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 9071500011277554477L;
@Override
public ATypeTag getTypeTag() {
@@ -936,7 +936,7 @@ public abstract class BuiltinType implements IAType {
};
public static final BuiltinType AUUID = new LowerCaseConstructorType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -8403791895909921192L;
@Override
public ATypeTag getTypeTag() {
@@ -969,7 +969,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType ANY = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -4041457851203593309L;
@Override
public ATypeTag getTypeTag() {
@@ -1007,7 +1007,7 @@ public abstract class BuiltinType implements IAType {
public static final BuiltinType SHORTWITHOUTTYPEINFO = new BuiltinType() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1769780932699820665L;
@Override
public ATypeTag getTypeTag() {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index 7e1ca61..d782ac5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -84,7 +84,7 @@ public class JobManager implements IJobManager {
}
activeRunMap = new HashMap<>();
runMapArchive = new LinkedHashMap<JobId, JobRun>() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = -1406441385508773629L;
@Override
protected boolean removeEldestEntry(Map.Entry<JobId, JobRun> eldest) {
@@ -92,7 +92,7 @@ public class JobManager implements IJobManager {
}
};
runMapHistory = new LinkedHashMap<JobId, List<Exception>>() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 7572062687032652986L;
/** history size + 1 is for the case when history size = 0 */
private final int allowedSize = 100 * (ccConfig.getJobHistorySize() + 1);
[asterixdb] 05/38: [NO ISSUE][HYR][MISC] Elimate hashtable for
thread name save/restore
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ef5467be498a284dde179973a5d9f404cd0c95e9
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Apr 15 18:22:57 2021 -0400
[NO ISSUE][HYR][MISC] Elimate hashtable for thread name save/restore
Change-Id: Ifeea72fb253601c214a18e8f4053d7f7d2b31135
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11084
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../hyracks/util/MaintainedThreadNameExecutorService.java | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
index a9ebb50..9adac04 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MaintainedThreadNameExecutorService.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.util;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
@@ -28,7 +26,7 @@ import java.util.concurrent.TimeUnit;
public class MaintainedThreadNameExecutorService extends ThreadPoolExecutor {
- private final Map<Thread, String> threadNames = new ConcurrentHashMap<>();
+ private static final ThreadLocal<String> savedName = new ThreadLocal<>();
private MaintainedThreadNameExecutorService(ThreadFactory threadFactory) {
super(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory);
@@ -40,17 +38,13 @@ public class MaintainedThreadNameExecutorService extends ThreadPoolExecutor {
@Override
protected void beforeExecute(Thread t, Runnable r) {
- threadNames.put(t, t.getName());
+ savedName.set(t.getName());
super.beforeExecute(t, r);
}
@Override
protected void afterExecute(Runnable r, Throwable t) {
super.afterExecute(r, t);
- final Thread thread = Thread.currentThread();
- final String originalThreadName = threadNames.remove(thread);
- if (originalThreadName != null) {
- thread.setName(originalThreadName);
- }
+ Thread.currentThread().setName(savedName.get());
}
}
[asterixdb] 20/38: [ASTERIXDB-2883][COMP] Improve null handling in
UDF calls
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 404e382c28d50e7008ef1c1d5b7095412f9de5d8
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon May 3 17:42:16 2021 -0700
[ASTERIXDB-2883][COMP] Improve null handling in UDF calls
- user model changes: no
- storage format changes: no
- interface changes: no
- IntroduceDynamicTypeCastForExternalFunctionRule
should not apply twice on the same operator
- Add null-call handling to ExternalTypeComputer
and ExternalScalarJavaFunctionEvaluator
Change-Id: I9c58e28f673606c5d0413bdc4cd3ba0c7c20eb8d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11306
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ian Maxon <im...@uci.edu>
---
...duceDynamicTypeCastForExternalFunctionRule.java | 9 ++++-
.../type_name/type_name.0.ddl.sqlpp | 20 ++++++++++
.../type_name/type_name.1.lib.sqlpp | 20 ++++++++++
.../type_name/type_name.2.ddl.sqlpp | 25 ++++++++++++
.../type_name/type_name.3.query.sqlpp | 28 +++++++++++++
.../type_name/type_name.4.ddl.sqlpp | 20 ++++++++++
.../external-library/type_name/type_name.3.adm | 1 +
.../resources/runtimets/testsuite_it_sqlpp.xml | 5 +++
.../ExternalScalarJavaFunctionEvaluator.java | 45 ++++++++++++++-------
.../asterix/external/library/TypeNameFactory.java | 30 ++++++++++++++
.../asterix/external/library/TypeNameFunction.java | 46 ++++++++++++++++++++++
.../functions/ExternalFunctionCompilerUtil.java | 2 +-
.../metadata/functions/ExternalTypeComputer.java | 13 ++++--
13 files changed, 244 insertions(+), 20 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
index d3eb0c2..2c5a07d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
@@ -115,6 +115,13 @@ public class IntroduceDynamicTypeCastForExternalFunctionRule implements IAlgebra
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
return false;
}
- return op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ boolean applied = op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (applied) {
+ context.addToDontApplySet(this, op);
+ }
+ return applied;
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
new file mode 100644
index 0000000..76cc70d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
new file mode 100644
index 0000000..b2bf929
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
new file mode 100644
index 0000000..c7d6768
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+create function typeName(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib;
+
+create function typeNameNullCall(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
new file mode 100644
index 0000000..2542d55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+{
+ "missing": typeName(missing) is missing,
+ "missing_nullcall": typeNameNullCall(missing),
+ "null": typeName(null) is null,
+ "null_nullcall": typeNameNullCall(null),
+ "boolean": typeName(true),
+ "string": typeName("x")
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
new file mode 100644
index 0000000..2b27030
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
new file mode 100644
index 0000000..a72c46c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
@@ -0,0 +1 @@
+{ "missing": true, "missing_nullcall": "missing", "null": true, "null_nullcall": "null", "boolean": "boolean", "string": "string" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 28cbabb..6c67216 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -42,6 +42,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
+ <compilation-unit name="type_name">
+ <output-dir compare="Text">type_name</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
<compilation-unit name="type_validation">
<output-dir compare="Text">type_validation</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index 33b0369..35d55f7 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -19,7 +19,7 @@
package org.apache.asterix.external.library;
-import java.io.IOException;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
@@ -27,7 +27,9 @@ import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -71,23 +73,36 @@ class ExternalScalarJavaFunctionEvaluator extends ExternalScalarFunctionEvaluato
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
try {
- setArguments(tuple);
- resultBuffer.reset();
- externalFunctionInstance.evaluate(functionHelper);
- if (!functionHelper.isValidResult()) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
+ for (int i = 0; i < argEvals.length; i++) {
+ argEvals[i].evaluate(tuple, inputVal);
+ if (!nullCall) {
+ byte[] inputValBytes = inputVal.getByteArray();
+ int inputValStartOffset = inputVal.getStartOffset();
+ ATypeTag typeTag = ATYPETAGDESERIALIZER.deserialize(inputValBytes[inputValStartOffset]);
+ if (typeTag == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (typeTag == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
+ }
+ functionHelper.setArgument(i, inputVal);
+ }
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ } else {
+ resultBuffer.reset();
+ externalFunctionInstance.evaluate(functionHelper);
+ if (!functionHelper.isValidResult()) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ }
+ result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
+ functionHelper.reset();
}
- result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
- functionHelper.reset();
} catch (Exception e) {
throw HyracksDataException.create(e);
}
}
-
- public void setArguments(IFrameTupleReference tuple) throws IOException {
- for (int i = 0; i < argEvals.length; i++) {
- argEvals[i].evaluate(tuple, inputVal);
- functionHelper.setArgument(i, inputVal);
- }
- }
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
new file mode 100644
index 0000000..335c383
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionFactory;
+
+public class TypeNameFactory implements IFunctionFactory {
+ @Override
+ public IExternalScalarFunction getExternalFunction() {
+ return new TypeNameFunction();
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
new file mode 100644
index 0000000..1f03fcc
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionHelper;
+import org.apache.asterix.external.library.java.base.JString;
+
+public class TypeNameFunction implements IExternalScalarFunction {
+
+ private JString result;
+
+ @Override
+ public void deinitialize() {
+ // nothing to do here
+ }
+
+ @Override
+ public void evaluate(IFunctionHelper functionHelper) throws Exception {
+ String arg0TypeName = functionHelper.getArgument(0).getIAType().getTypeName();
+ result.setValue(arg0TypeName);
+ functionHelper.setResult(result);
+ }
+
+ @Override
+ public void initialize(IFunctionHelper functionHelper) {
+ result = (JString) functionHelper.getResultObject();
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 6751171..5fd96a6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -69,7 +69,7 @@ public class ExternalFunctionCompilerUtil {
IAType returnType = getType(function.getReturnType(), metadataProvider);
- IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes);
+ IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes, function.getNullCall());
ExternalFunctionLanguage lang = getExternalFunctionLanguage(function.getLanguage());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
index 4a000a3..99d8ea2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
@@ -33,8 +33,9 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
public class ExternalTypeComputer extends AbstractResultTypeComputer {
- private IAType resultType;
- private List<IAType> paramPrimeTypes;
+ private final IAType resultType;
+ private final List<IAType> paramPrimeTypes;
+ private final boolean nullCall;
@Override
protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
@@ -47,14 +48,20 @@ public class ExternalTypeComputer extends AbstractResultTypeComputer {
}
}
- public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes) {
+ public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes, boolean nullCall) {
this.resultType = resultPrimeType.getTypeTag() == ATypeTag.ANY ? resultPrimeType
: AUnionType.createUnknownableType(resultPrimeType);
this.paramPrimeTypes = paramPrimeTypes;
+ this.nullCall = nullCall;
}
@Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) {
return resultType;
}
+
+ @Override
+ protected boolean propagateNullAndMissing() {
+ return !nullCall;
+ }
}
[asterixdb] 24/38: [NO ISSUE][EXT] Refactor JSONDataParser
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 91f91d25427257fac34ec17386f2ad5113c08242
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu May 6 16:06:47 2021 -0400
[NO ISSUE][EXT] Refactor JSONDataParser
Change-Id: Ia47f2fe449cfb0ca8e6e3a6308ef01ee6810ec03
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11363
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
...DataParser.java => AbstractJsonDataParser.java} | 78 +---
.../asterix/external/parser/JSONDataParser.java | 435 +--------------------
2 files changed, 27 insertions(+), 486 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
similarity index 85%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
index b2036c0..a5d5a08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -20,7 +20,6 @@ package org.apache.asterix.external.parser;
import java.io.DataOutput;
import java.io.IOException;
-import java.io.InputStream;
import java.util.BitSet;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
@@ -29,9 +28,6 @@ import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.IAsterixListBuilder;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.external.api.IRawRecord;
-import org.apache.asterix.external.api.IRecordDataParser;
-import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.parser.jackson.ADMToken;
import org.apache.asterix.external.parser.jackson.GeometryCoParser;
import org.apache.asterix.external.parser.jackson.ParserContext;
@@ -56,21 +52,18 @@ import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonStreamContext;
import com.fasterxml.jackson.core.JsonToken;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.TreeTraversingParser;
/**
- * JSON format parser using Jakson parser.
+ * JSON format parser using Jackson parser.
*/
-public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
- implements IStreamDataParser, IRecordDataParser<char[]> {
+public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<ADMToken> {
protected final ParserContext parserContext;
protected final JsonFactory jsonFactory;
protected final ARecordType rootType;
protected final GeometryCoParser geometryCoParser;
- private Supplier<String> dataSourceName;
- private LongSupplier lineNumber;
+ protected Supplier<String> dataSourceName = ExternalDataConstants.EMPTY_STRING;
+ protected LongSupplier lineNumber = ExternalDataConstants.NO_LINES;
protected JsonParser jsonParser;
@@ -82,15 +75,13 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
* @param jsonFactory
* Jackson JSON parser factory.
*/
- public JSONDataParser(ARecordType recordType, JsonFactory jsonFactory) {
+ public AbstractJsonDataParser(ARecordType recordType, JsonFactory jsonFactory) {
// recordType currently cannot be null, however this is to guarantee for any future changes.
this.rootType = recordType != null ? recordType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
this.jsonFactory = jsonFactory;
- //GeometyCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
+ //GeometryCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
geometryCoParser = new GeometryCoParser(jsonParser);
parserContext = new ParserContext();
- this.dataSourceName = ExternalDataConstants.EMPTY_STRING;
- this.lineNumber = ExternalDataConstants.NO_LINES;
}
/*
@@ -99,53 +90,6 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
****************************************************
*/
- @Override
- public void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
- this.dataSourceName = dataSourceName == null ? ExternalDataConstants.EMPTY_STRING : dataSourceName;
- this.lineNumber = lineNumber == null ? ExternalDataConstants.NO_LINES : lineNumber;
- }
-
- @Override
- public final boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
- try {
- //TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
- jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
- geometryCoParser.reset(jsonParser);
- nextToken();
- parseObject(rootType, out);
- return true;
- } catch (IOException e) {
- throw createException(e);
- }
- }
-
- @Override
- public void setInputStream(InputStream in) throws IOException {
- setInput(jsonFactory.createParser(in));
- }
-
- public void setInputNode(JsonNode node) {
- setInput(new TreeTraversingParser(node));
- }
-
- private void setInput(JsonParser parser) {
- jsonParser = parser;
- geometryCoParser.reset(jsonParser);
- }
-
- @Override
- public boolean parse(DataOutput out) throws HyracksDataException {
- try {
- if (nextToken() == ADMToken.EOF) {
- return false;
- }
- parseObject(rootType, out);
- return true;
- } catch (IOException e) {
- throw new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
- }
- }
-
public boolean parseAnyValue(DataOutput out) throws HyracksDataException {
try {
if (nextToken() == ADMToken.EOF) {
@@ -158,12 +102,6 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
}
}
- @Override
- public boolean reset(InputStream in) throws IOException {
- setInputStream(in);
- return true;
- }
-
/*
****************************************************
* Abstract method implementation
@@ -361,7 +299,7 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
* @param out
* @throws IOException
*/
- private void parseObject(IAType actualType, DataOutput out) throws IOException {
+ protected void parseObject(IAType actualType, DataOutput out) throws IOException {
if (actualType.getTypeTag() == ATypeTag.OBJECT) {
parseObject((ARecordType) actualType, out);
} else {
@@ -477,7 +415,7 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
}
}
- private HyracksDataException createException(IOException e) {
+ protected HyracksDataException createException(IOException e) {
if (jsonParser != null) {
String msg;
if (e instanceof JsonParseException) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
index b2036c0..b4ec46e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
@@ -21,61 +21,31 @@ package org.apache.asterix.external.parser;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
-import java.util.BitSet;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
-import org.apache.asterix.builders.IARecordBuilder;
-import org.apache.asterix.builders.IAsterixListBuilder;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.parser.jackson.ADMToken;
-import org.apache.asterix.external.parser.jackson.GeometryCoParser;
-import org.apache.asterix.external.parser.jackson.ParserContext;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.base.ANull;
-import org.apache.asterix.om.base.AUnorderedList;
-import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.AbstractCollectionType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.exceptions.UnsupportedTypeException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.util.ExceptionUtils;
-import org.apache.hyracks.data.std.api.IMutableValueStorage;
import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonStreamContext;
-import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.TreeTraversingParser;
/**
- * JSON format parser using Jakson parser.
+ * JSON format parser using Jackson parser.
*/
-public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
- implements IStreamDataParser, IRecordDataParser<char[]> {
-
- protected final ParserContext parserContext;
- protected final JsonFactory jsonFactory;
- protected final ARecordType rootType;
- protected final GeometryCoParser geometryCoParser;
- private Supplier<String> dataSourceName;
- private LongSupplier lineNumber;
-
- protected JsonParser jsonParser;
+public class JSONDataParser extends AbstractJsonDataParser implements IStreamDataParser, IRecordDataParser<char[]> {
/**
- * Initialize JSONDataParser with GeometryCoParser
+ * Initialize JSONDataParser
*
* @param recordType
* defined type.
@@ -83,40 +53,7 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
* Jackson JSON parser factory.
*/
public JSONDataParser(ARecordType recordType, JsonFactory jsonFactory) {
- // recordType currently cannot be null, however this is to guarantee for any future changes.
- this.rootType = recordType != null ? recordType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
- this.jsonFactory = jsonFactory;
- //GeometyCoParser to parse GeoJSON objects to AsterixDB internal spatial types.
- geometryCoParser = new GeometryCoParser(jsonParser);
- parserContext = new ParserContext();
- this.dataSourceName = ExternalDataConstants.EMPTY_STRING;
- this.lineNumber = ExternalDataConstants.NO_LINES;
- }
-
- /*
- ****************************************************
- * Public methods
- ****************************************************
- */
-
- @Override
- public void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
- this.dataSourceName = dataSourceName == null ? ExternalDataConstants.EMPTY_STRING : dataSourceName;
- this.lineNumber = lineNumber == null ? ExternalDataConstants.NO_LINES : lineNumber;
- }
-
- @Override
- public final boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
- try {
- //TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
- jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
- geometryCoParser.reset(jsonParser);
- nextToken();
- parseObject(rootType, out);
- return true;
- } catch (IOException e) {
- throw createException(e);
- }
+ super(recordType, jsonFactory);
}
@Override
@@ -134,24 +71,32 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
}
@Override
- public boolean parse(DataOutput out) throws HyracksDataException {
+ public boolean parse(IRawRecord<? extends char[]> record, DataOutput out) throws HyracksDataException {
try {
- if (nextToken() == ADMToken.EOF) {
- return false;
- }
+ //TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
+ jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
+ geometryCoParser.reset(jsonParser);
+ nextToken();
parseObject(rootType, out);
return true;
} catch (IOException e) {
- throw new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
+ throw createException(e);
}
}
- public boolean parseAnyValue(DataOutput out) throws HyracksDataException {
+ @Override
+ public void configure(Supplier<String> dataSourceName, LongSupplier lineNumber) {
+ this.dataSourceName = dataSourceName == null ? ExternalDataConstants.EMPTY_STRING : dataSourceName;
+ this.lineNumber = lineNumber == null ? ExternalDataConstants.NO_LINES : lineNumber;
+ }
+
+ @Override
+ public boolean parse(DataOutput out) throws HyracksDataException {
try {
if (nextToken() == ADMToken.EOF) {
return false;
}
- parseValue(BuiltinType.ANY, out);
+ parseObject(rootType, out);
return true;
} catch (IOException e) {
throw new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
@@ -163,346 +108,4 @@ public class JSONDataParser extends AbstractNestedDataParser<ADMToken>
setInputStream(in);
return true;
}
-
- /*
- ****************************************************
- * Abstract method implementation
- ****************************************************
- */
-
- /**
- * Jackson token to ADM token mapper
- */
- @Override
- protected final ADMToken advanceToNextToken() throws IOException {
- final JsonToken jsonToken = jsonParser.nextToken();
- if (jsonToken == null) {
- return ADMToken.EOF;
- }
- ADMToken token;
- switch (jsonToken) {
- case VALUE_FALSE:
- token = ADMToken.FALSE;
- break;
- case VALUE_TRUE:
- token = ADMToken.TRUE;
- break;
- case VALUE_STRING:
- token = ADMToken.STRING;
- break;
- case VALUE_NULL:
- token = ADMToken.NULL;
- break;
- case VALUE_NUMBER_FLOAT:
- token = ADMToken.DOUBLE;
- break;
- case VALUE_NUMBER_INT:
- token = ADMToken.INT;
- break;
- case START_OBJECT:
- token = ADMToken.OBJECT_START;
- break;
- case END_OBJECT:
- token = ADMToken.OBJECT_END;
- break;
- case START_ARRAY:
- token = ADMToken.ARRAY_START;
- break;
- case END_ARRAY:
- token = ADMToken.ARRAY_END;
- break;
- case FIELD_NAME:
- token = ADMToken.FIELD_NAME;
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
- }
-
- return token;
- }
- /*
- ****************************************************
- * Overridden methods
- ****************************************************
- */
-
- /**
- * In the case of JSON, we can parse GeoJSON objects as internal AsterixDB spatial types.
- */
- @Override
- protected boolean isConvertable(ATypeTag parsedTypeTag, ATypeTag definedTypeTag) {
- if (parsedTypeTag == ATypeTag.OBJECT && (definedTypeTag == ATypeTag.POINT || definedTypeTag == ATypeTag.LINE
- || definedTypeTag == ATypeTag.POLYGON)) {
- return true;
- }
- return super.isConvertable(parsedTypeTag, definedTypeTag);
- }
-
- /*
- ****************************************************
- * Complex types parsers
- ****************************************************
- */
-
- @Override
- protected final void parseObject(ARecordType recordType, DataOutput out) throws IOException {
- final IMutableValueStorage valueBuffer = parserContext.enterObject();
- final IARecordBuilder objectBuilder = parserContext.getObjectBuilder(recordType);
- final BitSet nullBitMap = parserContext.getNullBitmap(recordType.getFieldTypes().length);
- while (nextToken() != ADMToken.OBJECT_END) {
- /*
- * Jackson parser calls String.intern() for field names (if enabled).
- * Calling getCurrentName() will not create multiple objects.
- */
- final String fieldName = jsonParser.getCurrentName();
- final int fieldIndex = recordType.getFieldIndex(fieldName);
-
- if (!recordType.isOpen() && fieldIndex < 0) {
- throw new RuntimeDataException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD,
- fieldName);
- }
- valueBuffer.reset();
- nextToken();
-
- if (fieldIndex < 0) {
- //field is not defined and the type is open
- parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
- objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), valueBuffer);
- } else {
- //field is defined
- final IAType fieldType = recordType.getFieldType(fieldName);
-
- //fail fast if the current field is not nullable
- if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL, fieldName);
- }
-
- nullBitMap.set(fieldIndex);
- parseValue(fieldType, valueBuffer.getDataOutput());
- objectBuilder.addField(fieldIndex, valueBuffer);
- }
- }
-
- /*
- * Check for any possible missed values for a defined (non-nullable) type.
- * Throws exception if there is a violation
- */
- if (nullBitMap != null) {
- checkOptionalConstraints(recordType, nullBitMap);
- }
- parserContext.exitObject(valueBuffer, nullBitMap, objectBuilder);
- objectBuilder.write(out, true);
- }
-
- /**
- * Geometry in GeoJSON is an object
- *
- * @param typeTag
- * geometry typeTag
- * @param out
- * @throws IOException
- */
- private void parseGeometry(ATypeTag typeTag, DataOutput out) throws IOException {
- //Start the co-parser
- geometryCoParser.starGeometry();
- while (nextToken() != ADMToken.OBJECT_END) {
- if (currentToken() == ADMToken.FIELD_NAME) {
- geometryCoParser.checkFieldName(jsonParser.getCurrentName());
- } else if (!geometryCoParser.checkValue(currentToken())) {
- throw new IOException(geometryCoParser.getErrorMessage());
- }
- }
-
- geometryCoParser.serialize(typeTag, out);
- }
-
- @Override
- protected final void parseArray(AOrderedListType listType, DataOutput out) throws IOException {
- parseCollection(listType, ADMToken.ARRAY_END, out);
- }
-
- @Override
- protected void parseMultiset(AUnorderedList listType, DataOutput out) throws IOException {
- throw new UnsupportedTypeException("JSON parser", ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- protected final void parseCollection(AbstractCollectionType collectionType, ADMToken endToken, DataOutput out)
- throws IOException {
- final IMutableValueStorage valueBuffer = parserContext.enterCollection();
- final IAsterixListBuilder arrayBuilder = parserContext.getCollectionBuilder(collectionType);
- final boolean isOpen = collectionType.getItemType().getTypeTag() == ATypeTag.ANY;
- while (nextToken() != endToken) {
- valueBuffer.reset();
- if (isOpen) {
- parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
- } else {
- //fail fast if current value is null
- if (currentToken() == ADMToken.NULL) {
- throw new RuntimeDataException(ErrorCode.PARSER_COLLECTION_ITEM_CANNOT_BE_NULL);
- }
- parseValue(collectionType.getItemType(), valueBuffer.getDataOutput());
- }
- arrayBuilder.addItem(valueBuffer);
- }
- parserContext.exitCollection(valueBuffer, arrayBuilder);
- arrayBuilder.write(out, true);
- }
-
- /*
- ****************************************************
- * Value parsers and serializers
- ****************************************************
- */
-
- /**
- * Parse JSON object or GeoJSON object.
- *
- * @param actualType
- * @param out
- * @throws IOException
- */
- private void parseObject(IAType actualType, DataOutput out) throws IOException {
- if (actualType.getTypeTag() == ATypeTag.OBJECT) {
- parseObject((ARecordType) actualType, out);
- } else {
- parseGeometry(actualType.getTypeTag(), out);
- }
- }
-
- protected void parseValue(IAType definedType, DataOutput out) throws IOException {
- final ATypeTag currentTypeTag = currentToken().getTypeTag();
- /*
- * In case of type mismatch, checkAndGetType will throw an exception.
- */
- final IAType actualType = checkAndGetType(definedType, currentTypeTag);
-
- switch (currentToken()) {
- case NULL:
- nullSerde.serialize(ANull.NULL, out);
- break;
- case FALSE:
- booleanSerde.serialize(ABoolean.FALSE, out);
- break;
- case TRUE:
- booleanSerde.serialize(ABoolean.TRUE, out);
- break;
- case INT:
- case DOUBLE:
- serailizeNumeric(actualType.getTypeTag(), out);
- break;
- case STRING:
- serializeString(actualType.getTypeTag(), out);
- break;
- case OBJECT_START:
- parseObject(actualType, out);
- break;
- case ARRAY_START:
- parseArray((AOrderedListType) actualType, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.PARSE_ERROR, jsonParser.currentToken().toString());
- }
- }
-
- /**
- * Given that numeric values may underflow or overflow, an exception will be thrown.
- *
- * @param numericType
- * @param out
- * @throws IOException
- */
- private void serailizeNumeric(ATypeTag numericType, DataOutput out) throws IOException {
- final ATypeTag typeToUse = numericType == ATypeTag.ANY ? currentToken().getTypeTag() : numericType;
-
- switch (typeToUse) {
- case BIGINT:
- aInt64.setValue(jsonParser.getLongValue());
- int64Serde.serialize(aInt64, out);
- break;
- case INTEGER:
- aInt32.setValue(jsonParser.getIntValue());
- int32Serde.serialize(aInt32, out);
- break;
- case SMALLINT:
- aInt16.setValue(jsonParser.getShortValue());
- int16Serde.serialize(aInt16, out);
- break;
- case TINYINT:
- aInt8.setValue(jsonParser.getByteValue());
- int8Serde.serialize(aInt8, out);
- break;
- case DOUBLE:
- aDouble.setValue(jsonParser.getDoubleValue());
- doubleSerde.serialize(aDouble, out);
- break;
- case FLOAT:
- aFloat.setValue(jsonParser.getFloatValue());
- floatSerde.serialize(aFloat, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
- }
- }
-
- /**
- * Serialize the string value.
- * TODO(wyk) avoid String objects for type STRING
- *
- * @param stringVariantType
- * @param out
- * @throws IOException
- */
- private void serializeString(ATypeTag stringVariantType, DataOutput out) throws IOException {
- char[] buffer = jsonParser.getTextCharacters();
- int begin = jsonParser.getTextOffset();
- int len = jsonParser.getTextLength();
- final ATypeTag typeToUse = stringVariantType == ATypeTag.ANY ? currentToken().getTypeTag() : stringVariantType;
-
- switch (typeToUse) {
- case STRING:
- parseString(buffer, begin, len, out);
- break;
- case DATE:
- parseDate(buffer, begin, len, out);
- break;
- case DATETIME:
- parseDateTime(buffer, begin, len, out);
- break;
- case TIME:
- parseTime(buffer, begin, len, out);
- break;
- default:
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, jsonParser.currentToken().toString());
-
- }
- }
-
- private HyracksDataException createException(IOException e) {
- if (jsonParser != null) {
- String msg;
- if (e instanceof JsonParseException) {
- msg = ((JsonParseException) e).getOriginalMessage();
- } else {
- msg = ExceptionUtils.getRootCause(e).getMessage();
- }
- if (msg == null) {
- msg = ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM.errorMessage();
- }
- long lineNum = lineNumber.getAsLong() + jsonParser.getCurrentLocation().getLineNr() - 1;
- JsonStreamContext parsingContext = jsonParser.getParsingContext();
- String fieldName = "N/A";
- while (parsingContext != null) {
- String currentFieldName = parsingContext.getCurrentName();
- if (currentFieldName != null) {
- fieldName = currentFieldName;
- break;
- }
- parsingContext = parsingContext.getParent();
- }
-
- return HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.PARSING_ERROR,
- dataSourceName.get(), lineNum, fieldName, msg);
- }
- return new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
- }
}
[asterixdb] 27/38: [NO ISSUE][TX] Ensure lock manager computes
valid indexes for hash table
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ed56b863625ec6e9ed05ca58e13329e332ebe457
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Mon May 10 19:53:45 2021 +0300
[NO ISSUE][TX] Ensure lock manager computes valid indexes for hash table
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Given a dataset id and an entity hash value, ensure that the lock
manager computes a valid index for looking up the hash table.
Change-Id: I33d342455a79d0d69b49e126ff6a40c47b07d0dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11403
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../transaction/management/service/locking/ResourceGroupTable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
index 93cda83..a82b302 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
@@ -41,7 +41,7 @@ class ResourceGroupTable {
ResourceGroup get(int dId, int entityHashValue) {
// TODO ensure good properties of hash function
- return table[Math.abs(dId ^ entityHashValue) % size];
+ return table[Math.abs((dId ^ entityHashValue) % size)];
}
ResourceGroup get(int i) {
[asterixdb] 22/38: [NO ISSUE][OTH] Only order nodes in node group
for new datasets
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 30c4a78fcecf8d81a3cc070e348a38cae5600d1a
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu May 6 01:39:41 2021 +0300
[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/app/function/DumpIndexDatasource.java | 14 +++++++++++---
.../apache/asterix/app/function/DumpIndexRewriter.java | 6 +++++-
.../apache/asterix/metadata/entities/NodeGroup.java | 18 +++++++++---------
.../org/apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../metadata/utils/SecondaryIndexOperationsHelper.java | 4 ++++
.../asterix/runtime/utils/ClusterStateManager.java | 2 --
6 files changed, 30 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@ public class DumpIndexDatasource extends FunctionDataSource {
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@ public class DumpIndexRewriter extends FunctionRewriter {
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@ package org.apache.asterix.metadata.entities;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@ import org.apache.asterix.metadata.api.IMetadataEntity;
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@ public class DatasetUtil {
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@ public abstract class SecondaryIndexOperationsHelper {
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@ public class ClusterStateManager implements IClusterStateManager {
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}
[asterixdb] 10/38: [NO ISSUE][API] Make the AbstractServlet look
for /* wild card
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0d88d1187acfe1902e766e1bfeec6c3d2d3384a5
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Apr 22 17:46:12 2021 +0300
[NO ISSUE][API] Make the AbstractServlet look for /* wild card
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Make the AbstractServlet look for /* wild card to determine
the servlet path length
Change-Id: I74d393ede9e3bcf91222e2aa0e4a501a48ac0cf3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11144
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../api/http/server/AbstractNCUdfServlet.java | 2 +-
.../hyracks/http/server/AbstractServlet.java | 23 +++++++++++-----------
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
index cf287d9..5a3cde6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
@@ -134,7 +134,7 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet {
}
URI createDownloadURI(Path file) throws Exception {
- String path = paths[0].substring(0, trims[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
+ String path = paths[0].substring(0, servletPathLengths[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
String host = getHyracksClientConnection().getHost();
return new URI(httpServerProtocol.toString(), null, host, httpServerPort, path, null, null);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
index 514a7dd..ed567f5 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/AbstractServlet.java
@@ -53,20 +53,20 @@ public abstract class AbstractServlet implements IServlet {
protected final String[] paths;
protected final ConcurrentMap<String, Object> ctx;
- protected final int[] trims;
+ protected final int[] servletPathLengths;
public AbstractServlet(ConcurrentMap<String, Object> ctx, String... paths) {
this.paths = paths;
this.ctx = ctx;
- trims = new int[paths.length];
+ servletPathLengths = new int[paths.length];
for (int i = 0; i < paths.length; i++) {
String path = paths[i];
if (path.endsWith("/*")) {
- trims[i] = path.indexOf("/*");
+ servletPathLengths[i] = path.indexOf("/*");
} else if (path.endsWith("/")) {
- trims[i] = path.length() - 1;
+ servletPathLengths[i] = path.length() - 1;
} else {
- trims[i] = path.length();
+ servletPathLengths[i] = path.length();
}
}
}
@@ -175,26 +175,27 @@ public abstract class AbstractServlet implements IServlet {
public String localPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
int queryStart = uri.indexOf('?');
- return queryStart == -1 ? uri.substring(trim(uri)) : uri.substring(trim(uri), queryStart);
+ return queryStart == -1 ? uri.substring(servletLength(uri)) : uri.substring(servletLength(uri), queryStart);
}
public String servletPath(IServletRequest request) {
final String uri = request.getHttpRequest().uri();
- return uri.substring(0, trim(uri));
+ return uri.substring(0, servletLength(uri));
}
- protected int trim(final String uri) {
+ protected int servletLength(final String uri) {
int trim = -1;
if (paths.length > 1) {
for (int i = 0; i < paths.length; i++) {
- String path = paths[i].indexOf('*') >= 0 ? paths[i].substring(0, paths[i].indexOf('*')) : paths[i];
+ int wildCardIdx = paths[i].indexOf("/*");
+ String path = wildCardIdx >= 0 ? paths[i].substring(0, wildCardIdx) : paths[i];
if (uri.indexOf(path) == 0) {
- trim = trims[i];
+ trim = servletPathLengths[i];
break;
}
}
} else {
- trim = trims[0];
+ trim = servletPathLengths[0];
}
return trim;
}
[asterixdb] 03/38: [NO ISSUE][HYR][MISC] Clear interrupted state on
halt
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 33647b6519671b5074c158f822193ce2db41b7a0
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Apr 15 18:08:02 2021 -0400
[NO ISSUE][HYR][MISC] Clear interrupted state on halt
Clear the calling thread's interrupted state on halt, to enable collection of
thread dump at exit. Should the thread be interrupted again before the thread
dump has completed, the collection will be aborted and the halt will proceed.
Change-Id: I29584c059bdae32baff67ba6f6d435cc8458b107
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11083
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Michael Blow <mb...@apache.org>
---
.../src/main/java/org/apache/hyracks/util/ExitUtil.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index abd9fda..f4c4183 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -103,9 +103,11 @@ public class ExitUtil {
public static synchronized void halt(int status, Level logLevel) {
try {
+ boolean interrupted = Thread.interrupted();
+ LOGGER.log(logLevel, "JVM halting with status {} (halting thread {}, interrupted {})", status,
+ Thread.currentThread(), interrupted);
Future<?> future = haltThreadDumpExecutor.submit(() -> {
- LOGGER.log(logLevel, "JVM halting with status {}; thread dump at halt: {}", status,
- ThreadDumpUtil.takeDumpString());
+ LOGGER.log(logLevel, "Thread dump at halt: {}", ThreadDumpUtil.takeDumpString());
// try to give time for the log to be emitted...
LogManager.shutdown();
});
[asterixdb] 08/38: [NO ISSUE][OTH] Add operator field to subplan
json plan
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d5eb016761601e66a2fdd66c8cd8cc72cf248a03
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Apr 21 18:28:38 2021 +0300
[NO ISSUE][OTH] Add operator field to subplan json plan
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add "operator" field to subplan operators json format
plan.
Change-Id: I1df5a97367286663b002547997c555b20e56697d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11123
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../prettyprint/LogicalOperatorPrettyPrintVisitorJson.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
index f404058..6ba4c0c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitorJson.java
@@ -411,8 +411,13 @@ public class LogicalOperatorPrettyPrintVisitorJson extends AbstractLogicalOperat
@Override
public Void visitSubplanOperator(SubplanOperator op, Void indent) throws AlgebricksException {
- if (!op.getNestedPlans().isEmpty()) {
- writeNestedPlans(op, indent);
+ try {
+ if (!op.getNestedPlans().isEmpty()) {
+ jsonGenerator.writeStringField(OPERATOR_FIELD, "subplan");
+ writeNestedPlans(op, indent);
+ }
+ } catch (IOException e) {
+ throw AlgebricksException.create(ErrorCode.ERROR_PRINTING_PLAN, e, String.valueOf(e));
}
return null;
}
[asterixdb] 09/38: [NO ISSUE][MISC] Misc / minor diagnostic
improvement
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d5edad2eccb0a7fa0ca3e99f19b165deae2417b9
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Apr 22 08:50:01 2021 -0400
[NO ISSUE][MISC] Misc / minor diagnostic improvement
- include target runtimeId & message kind for ActiveManagerMessage.toString()
- add ThrowingFunction.asUnchecked() helper
Change-Id: I9e1cccb216033f08b69d3000a68dc8593704af9f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11143
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/active/message/ActiveManagerMessage.java | 2 +-
.../org/apache/hyracks/util/ThrowingFunction.java | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
index 4d726cf..bad3f79 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
@@ -64,6 +64,6 @@ public class ActiveManagerMessage extends CcIdentifiedMessage implements INcAddr
@Override
public String toString() {
- return getClass().getSimpleName();
+ return getClass().getSimpleName() + "{" + "kind=" + kind + ", runtimeId=" + runtimeId + '}';
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
index d18b41b..f60efbc 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingFunction.java
@@ -18,7 +18,26 @@
*/
package org.apache.hyracks.util;
+import java.util.function.Function;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
@FunctionalInterface
public interface ThrowingFunction<I, R> {
R process(I input) throws Exception;
+
+ @SuppressWarnings("Duplicates")
+ static <I, R> Function<I, R> asUnchecked(ThrowingFunction<I, R> consumer) {
+ return input -> {
+ try {
+ return consumer.process(input);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new UncheckedExecutionException(e);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ };
+ }
+
}
[asterixdb] 13/38: [NO ISSUE][HYR][HTTP] Set thread name on
interruptable stream handler
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 902f63c6683cee08c6340a6a6fe007277722d1be
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Apr 25 10:52:25 2021 -0400
[NO ISSUE][HYR][HTTP] Set thread name on interruptable stream handler
Change-Id: I26b718cb0373c6c4d04d7fb0666de472c0419b29
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11183
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../main/java/org/apache/hyracks/http/server/utils/HttpUtil.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 01baa69..4c0942b 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -191,11 +191,13 @@ public class HttpUtil {
}
public static void handleStreamInterruptibly(CloseableHttpResponse response,
- ThrowingConsumer<Reader> streamProcessor, ExecutorService executor, Supplier<String> taskDescription)
+ ThrowingConsumer<Reader> streamProcessor, ExecutorService executor, Supplier<String> descriptionSupplier)
throws IOException, InterruptedException, ExecutionException {
// we have to consume the stream in a separate thread, as it not stop on interrupt; we need to
// instead close the connection to achieve the interrupt
+ String description = descriptionSupplier.get();
Future<Void> readFuture = executor.submit(() -> {
+ Thread.currentThread().setName(description);
InputStreamReader reader = new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8);
streamProcessor.process(new Reader() {
@Override
@@ -219,10 +221,10 @@ public class HttpUtil {
try {
readFuture.get(1, TimeUnit.SECONDS);
} catch (TimeoutException te) {
- LOGGER.warn("{} did not exit on stream close due to interrupt after 1s", taskDescription);
+ LOGGER.warn("{} did not exit on stream close due to interrupt after 1s", description);
readFuture.cancel(true);
} catch (ExecutionException ee) {
- LOGGER.debug("ignoring exception awaiting aborted {} shutdown", taskDescription, ee);
+ LOGGER.debug("ignoring exception awaiting aborted {} shutdown", description, ee);
}
throw ex;
}
[asterixdb] 35/38: [NO ISSUE][*DB][EXT] Fail with helpful error
message on non-JSON object
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 528ee18a5a7d9c01aa1f3f94282fbff2063f688a
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 19 20:10:46 2021 -0400
[NO ISSUE][*DB][EXT] Fail with helpful error message on non-JSON object
Change-Id: Ia096e108aa062e87a92f29ad3812f118390f411e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11525
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../java/org/apache/asterix/common/exceptions/ErrorCode.java | 3 ++-
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../asterix/external/parser/AbstractJsonDataParser.java | 11 ++++++++---
.../asterix/external/parser/AbstractNestedDataParser.java | 2 +-
.../org/apache/asterix/external/parser/JSONDataParser.java | 6 +++++-
.../org/apache/asterix/external/parser/ParseException.java | 4 ++++
.../java/org/apache/asterix/external/parser/TweetParser.java | 2 +-
.../java/org/apache/hyracks/api/util/ErrorMessageUtil.java | 11 +++++++----
8 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 7e32e0b..4078ea2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -307,7 +307,7 @@ public enum ErrorCode implements IError {
PARSER_ADM_DATA_PARSER_CAST_ERROR(3072),
PARSER_ADM_DATA_PARSER_CONSTRUCTOR_MISSING_DESERIALIZER(3073),
PARSER_ADM_DATA_PARSER_WRONG_INSTANCE(3074),
- PARSER_TWEET_PARSER_CLOSED_FIELD_NULL(3075),
+ PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL(3075),
UTIL_FILE_SYSTEM_WATCHER_NO_FILES_FOUND(3076),
UTIL_LOCAL_FILE_SYSTEM_UTILS_PATH_NOT_FOUND(3077),
UTIL_HDFS_UTILS_CANNOT_OBTAIN_HDFS_SCHEDULER(3078),
@@ -349,6 +349,7 @@ public enum ErrorCode implements IError {
FAILED_TO_PARSE_MALFORMED_LOG_RECORD(3117),
ACTIVE_ENTITY_NOT_RUNNING(3118),
REQUIRED_PARAM_IF_PARAM_IS_PRESENT(3119),
+ PARSER_DATA_PARSER_UNEXPECTED_TOKEN(3120),
// Lifecycle management errors
DUPLICATE_PARTITION_ID(4000),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 345b9d8..18898bb 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -354,6 +354,7 @@
3117 = Failed to parse record, malformed log record
3118 = Active Entity %1$s is not running (it is %2$s)
3119 = Parameter '%1$s' is required if '%2$s' is provided
+3120 = Unexpected token %s: was expecting %s
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
index a5d5a08..2bf0df4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -212,7 +212,7 @@ public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<AD
//fail fast if the current field is not nullable
if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL, fieldName);
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, fieldName);
}
nullBitMap.set(fieldIndex);
@@ -415,13 +415,18 @@ public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<AD
}
}
- protected HyracksDataException createException(IOException e) {
+ protected HyracksDataException createException(Exception e) {
if (jsonParser != null) {
String msg;
if (e instanceof JsonParseException) {
msg = ((JsonParseException) e).getOriginalMessage();
} else {
- msg = ExceptionUtils.getRootCause(e).getMessage();
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof ParseException) {
+ msg = ((ParseException) rootCause).getOriginalMessage();
+ } else {
+ msg = ExceptionUtils.getRootCause(e).getMessage();
+ }
}
if (msg == null) {
msg = ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM.errorMessage();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
index c6f605d..eecbb19 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java
@@ -116,7 +116,7 @@ public abstract class AbstractNestedDataParser<T> extends AbstractDataParser {
protected void checkOptionalConstraints(ARecordType recordType, BitSet nullBitmap) throws RuntimeDataException {
for (int i = 0; i < recordType.getFieldTypes().length; i++) {
if (!nullBitmap.get(i) && !isMissableType(recordType.getFieldTypes()[i])) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL,
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
recordType.getFieldNames()[i]);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
index b4ec46e..b2cffa9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/JSONDataParser.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.external.parser;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARSER_DATA_PARSER_UNEXPECTED_TOKEN;
+
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
@@ -76,7 +78,9 @@ public class JSONDataParser extends AbstractJsonDataParser implements IStreamDat
//TODO(wyk): find a way to reset byte[] instead of creating a new parser for each record.
jsonParser = jsonFactory.createParser(record.get(), 0, record.size());
geometryCoParser.reset(jsonParser);
- nextToken();
+ if (nextToken() != ADMToken.OBJECT_START) {
+ throw new ParseException(PARSER_DATA_PARSER_UNEXPECTED_TOKEN, currentToken(), ADMToken.OBJECT_START);
+ }
parseObject(rootType, out);
return true;
} catch (IOException e) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
index e9f93c9..f130fb3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ParseException.java
@@ -84,4 +84,8 @@ public class ParseException extends HyracksDataException {
}
return msg.append(": ").append(super.getMessage()).toString();
}
+
+ public String getOriginalMessage() {
+ return super.getMessage();
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
index 4726a50..b970b04 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
@@ -199,7 +199,7 @@ public class TweetParser extends AbstractDataParser implements IRecordDataParser
DataOutput fieldOutput = fieldValueBuffer.getDataOutput();
if (obj.get(curFNames[iter1]).isNull() && !(curTypes[iter1] instanceof AUnionType)) {
if (curRecType.isClosedField(curFNames[iter1])) {
- throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL,
+ throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
curFNames[iter1]);
} else {
continue;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
index 6479c8d..70b13fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
@@ -102,12 +102,15 @@ public class ErrorMessageUtil {
try (Formatter fmt = new Formatter()) {
if (!NONE.equals(component)) {
fmt.format("%1$s%2$04d: ", component, errorCode);
+
+ // if the message is already formatted, just return it
+ if (message.startsWith(fmt.toString())) {
+ return message;
+ }
}
- // if the message is already formatted, just return it
- if (!fmt.toString().isEmpty() && message.startsWith(fmt.toString())) {
- return message;
+ if (message != null) {
+ fmt.format(message, (Object[]) params);
}
- fmt.format(message == null ? "null" : message, (Object[]) params);
if (sourceLoc != null) {
fmt.out().append(" (in line ").append(String.valueOf(sourceLoc.getLine())).append(", at column ")
.append(String.valueOf(sourceLoc.getColumn())).append(')');
[asterixdb] 21/38: [NO ISSUE][MISC] Avoid URI.getHost/Port where
hostnames may contain underscore
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d28f9addb8d4e0ea80bb3c88f828aa60d5e3783f
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon May 3 16:48:08 2021 -0400
[NO ISSUE][MISC] Avoid URI.getHost/Port where hostnames may contain underscore
Change-Id: Iff58db84cc4d11609e7d7c459003a8a50f058ac6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11305
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/app/nc/task/RetrieveLibrariesTask.java | 7 +++++--
.../asterix/app/external/ExternalUDFLibrarian.java | 3 ++-
.../apache/asterix/test/common/TestExecutor.java | 9 +++------
.../java/org/apache/hyracks/util/NetworkUtil.java | 22 ++++++++++++++++++++++
.../org/apache/hyracks/util/NetworkUtilTest.java | 11 +++++++++++
5 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
index 18d303c..715e626 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/RetrieveLibrariesTask.java
@@ -33,12 +33,14 @@ import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.client.utils.URIUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.control.CcId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.service.IControllerService;
+import org.apache.hyracks.util.NetworkUtil;
import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -51,7 +53,7 @@ public class RetrieveLibrariesTask implements INCLifecycleTask {
public RetrieveLibrariesTask(List<Pair<URI, String>> nodes) {
this.nodes = nodes;
- if (nodes.size() <= 0) {
+ if (nodes.isEmpty()) {
throw new IllegalArgumentException("No nodes specified to retrieve from");
}
}
@@ -62,7 +64,8 @@ public class RetrieveLibrariesTask implements INCLifecycleTask {
boolean success = false;
for (Pair<URI, String> referenceNode : nodes) {
try {
- LOGGER.info("Retrieving UDFs from " + referenceNode.getFirst().getHost());
+ LOGGER.info("Retrieving UDFs from "
+ + NetworkUtil.toHostPort(URIUtils.extractHost(referenceNode.getFirst())));
retrieveLibrary(referenceNode.getFirst(), referenceNode.getSecond(), appContext);
success = true;
break;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 2450025..e32f8ea 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -33,6 +33,7 @@ import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.client.utils.URIUtils;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
@@ -78,7 +79,7 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
private HttpClientContext createHttpClientContext(URI path, Pair<String, String> credentials) {
HttpClientContext hcCtx = HttpClientContext.create();
- HttpHost h = new HttpHost(path.getHost(), path.getPort(), "http");
+ HttpHost h = URIUtils.extractHost(path);
CredentialsProvider cp = new BasicCredentialsProvider();
cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(credentials.first, credentials.second));
hcCtx.setCredentialsProvider(cp);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 28b7fb3..40946d3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -106,7 +106,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
@@ -119,6 +118,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.client.utils.URIUtils;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
@@ -207,10 +207,7 @@ public class TestExecutor {
private static final int MAX_NON_UTF_8_STATEMENT_SIZE = 64 * 1024;
private static final ContentType TEXT_PLAIN_UTF8 = ContentType.create(HttpUtil.ContentType.APPLICATION_JSON, UTF_8);
- private final IPollTask plainExecutor = (testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
- queryCount, expectedResultFileCtxs, testFile, actualPath, expectedWarnings) -> executeTestFile(testCaseCtx,
- ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount, expectedResultFileCtxs,
- testFile, actualPath, expectedWarnings);
+ private final IPollTask plainExecutor = this::executeTestFile;
public static final String DELIVERY_ASYNC = "async";
public static final String DELIVERY_DEFERRED = "deferred";
@@ -675,7 +672,7 @@ public class TestExecutor {
cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(credentials.first, credentials.second));
HttpClientContext hcCtx = HttpClientContext.create();
AuthCache ac = new BasicAuthCache();
- ac.put(new HttpHost(method.getURI().getHost(), method.getURI().getPort(), "http"), new BasicScheme());
+ ac.put(URIUtils.extractHost(method.getURI()), new BasicScheme());
hcCtx.setAuthCache(ac);
CloseableHttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE)
.setDefaultCredentialsProvider(cp).build();
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
index 958310d..4f0c3a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/NetworkUtil.java
@@ -110,6 +110,28 @@ public class NetworkUtil {
return address != null ? toHostPort(address.getHostString(), address.getPort()) : null;
}
+ public static String toHostPort(HttpHost httpHost) {
+ if (httpHost == null) {
+ return null;
+ }
+ int port = httpHost.getPort();
+ if (port == -1) {
+ port = "https".equalsIgnoreCase(httpHost.getSchemeName()) ? 443 : 80;
+ }
+ return toHostPort(httpHost.getHostName(), port);
+ }
+
+ public static String toHostPort(HttpHost httpHost, int defaultPort) {
+ if (httpHost == null) {
+ return null;
+ }
+ int port = httpHost.getPort();
+ if (port == -1) {
+ port = defaultPort;
+ }
+ return toHostPort(httpHost.getHostName(), port);
+ }
+
public static InetSocketAddress parseInetSocketAddress(String hostPortString) {
int lastColon = hostPortString.lastIndexOf(':');
String host = decodeIPv6LiteralHost(lastColon < 0 ? hostPortString : hostPortString.substring(0, lastColon));
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
index c5d42c5..9300808 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/NetworkUtilTest.java
@@ -18,6 +18,9 @@
*/
package org.apache.hyracks.util;
+import java.net.URI;
+
+import org.apache.http.client.utils.URIUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Assert;
@@ -38,5 +41,13 @@ public class NetworkUtilTest {
Assert.assertEquals("localhost.localdomain.local:1234",
NetworkUtil.defaultPort("localhost.localdomain.local:1234", 9999));
+ Assert.assertEquals("[::1]:1234",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]:1234"))));
+ Assert.assertEquals("[::1]:80", NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]"))));
+ Assert.assertEquals("[::1]:443", NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("https://[::1]"))));
+ Assert.assertEquals("[::1]:1234",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("https://[::1]:1234")), 18091));
+ Assert.assertEquals("[::1]:8091",
+ NetworkUtil.toHostPort(URIUtils.extractHost(URI.create("http://[::1]")), 8091));
}
}
[asterixdb] 38/38: Merge branch 'cheshire-cat'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e7974ce1adf788c93868736704ce3d755d51de22
Merge: e75c8ef 3435172
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Fri May 21 12:41:46 2021 -0700
Merge branch 'cheshire-cat'
Change-Id: I72241c264b37ead5ae1fc40ed1f2b50f20c8689c
.../active/message/ActiveManagerMessage.java | 2 +-
.../ExtractBatchableExternalFunctionCallsRule.java | 12 +-
...duceDynamicTypeCastForExternalFunctionRule.java | 13 +-
.../asterix/translator/IRequestParameters.java | 2 +
.../asterix/translator/IStatementExecutor.java | 6 +-
.../apache/asterix/api/common/APIFramework.java | 24 +-
.../api/http/server/AbstractNCUdfServlet.java | 2 +-
.../app/active/ActiveEntityEventsListener.java | 7 +-
.../asterix/app/function/DumpIndexDatasource.java | 14 +-
.../asterix/app/function/DumpIndexRewriter.java | 6 +-
.../message/ExecuteStatementRequestMessage.java | 20 +-
.../org/apache/asterix/app/nc/RecoveryManager.java | 20 +-
.../asterix/app/nc/task/RetrieveLibrariesTask.java | 7 +-
.../asterix/app/translator/QueryTranslator.java | 16 +-
.../asterix/app/translator/RequestParameters.java | 17 +
.../org/apache/asterix/utils/FeedOperations.java | 2 +-
.../asterix/app/external/ExternalUDFLibrarian.java | 3 +-
.../apache/asterix/test/common/TestExecutor.java | 9 +-
.../test/dataflow/LSMFlushRecoveryTest.java | 12 +-
.../s3/anonymous_no_auth/test.000.ddl.sqlpp} | 18 +-
.../s3/anonymous_no_auth/test.001.ddl.sqlpp} | 18 +-
.../s3/anonymous_no_auth/test.002.ddl.sqlpp} | 17 +-
.../s3/anonymous_no_auth/test.099.ddl.sqlpp} | 4 +-
....query.sqlpp => type_validation.10.query.sqlpp} | 7 +-
...on.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} | 0
.../type_validation.2.ddl.sqlpp | 4 +-
.../type_validation.3.query.sqlpp | 3 +-
...3.query.sqlpp => type_validation.4.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.5.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.6.query.sqlpp} | 7 +-
...3.query.sqlpp => type_validation.7.query.sqlpp} | 3 +-
...3.query.sqlpp => type_validation.8.query.sqlpp} | 5 +-
...3.query.sqlpp => type_validation.9.query.sqlpp} | 5 +-
...ntiment.0.ddl.sqlpp => toplevel_fn.0.ddl.sqlpp} | 0
...ntiment.1.lib.sqlpp => toplevel_fn.1.lib.sqlpp} | 0
...ntiment.2.ddl.sqlpp => toplevel_fn.2.ddl.sqlpp} | 0
...ent.3.query.sqlpp => toplevel_fn.3.query.sqlpp} | 0
...ent.3.query.sqlpp => toplevel_fn.4.query.sqlpp} | 7 +-
...ntiment.4.ddl.sqlpp => toplevel_fn.5.ddl.sqlpp} | 0
.../type_name.0.ddl.sqlpp} | 0
.../type_name.1.lib.sqlpp} | 4 +-
.../type_name.2.ddl.sqlpp} | 6 +-
.../type_name.3.query.sqlpp} | 11 +-
.../type_name.4.ddl.sqlpp} | 0
.../query-ASTERIXDB-2886.1.ddl.sqlpp} | 25 +-
.../query-ASTERIXDB-2886.2.ddl.sqlpp} | 25 +-
.../query-ASTERIXDB-2886.3.query.sqlpp} | 14 +-
.../special_chars_2/special_chars_2.2.query.sqlpp | 1 -
.../type_validation.1.adm | 2 +-
.../type_validation.2.adm | 1 +
.../type_validation.3.adm | 1 +
.../type_validation.4.adm | 1 +
.../type_validation.5.adm | 1 +
.../type_validation.6.adm | 1 +
.../type_validation.7.adm | 1 +
.../type_validation.8.adm | 1 +
.../{toplevel_fn.1.adm => toplevel_fn.3.adm} | 0
.../external-library/toplevel_fn/toplevel_fn.4.adm | 1 +
.../external-library/type_name/type_name.3.adm | 1 +
.../push-limit-to-primary-scan-select.11.adm | 4 +-
.../push-limit-to-primary-scan.7.adm | 4 +-
.../query-ASTERIXDB-2886.3.adm | 4 +
.../special_chars_2/special_chars_2.2.adm | 12 +-
.../results/union/union_opt_1/union_opt_1.11.adm | 8 +-
...stsuite_external_dataset_azure_blob_storage.xml | 10 +-
.../runtimets/testsuite_external_dataset_s3.xml | 17 +-
...testsuite_external_dataset_s3_one_partition.xml | 20 +-
.../resources/runtimets/testsuite_it_sqlpp.xml | 5 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../common/api/IDatasetLifecycleManager.java | 9 +
.../common/context/DatasetLifecycleManager.java | 11 +
.../asterix/common/exceptions/ErrorCode.java | 4 +-
.../common/transactions/IRecoveryManager.java | 11 -
.../src/main/resources/asx_errormsg/en.properties | 2 +
.../src/main/user-defined_function/udf.md | 63 ++-
.../dataflow/AbstractFeedDataFlowController.java | 5 +
.../dataflow/FeedRecordDataFlowController.java | 5 +-
.../dataflow/FeedStreamDataFlowController.java | 1 -
.../reader/stream/QuotedLineRecordReader.java | 2 +-
.../reader/stream/SemiStructuredRecordReader.java | 2 +-
.../ExternalScalarJavaFunctionEvaluator.java | 45 ++-
.../ExternalScalarPythonFunctionEvaluator.java | 30 +-
.../ExternalAssignBatchRuntimeFactory.java | 19 +-
...DataParser.java => AbstractJsonDataParser.java} | 105 ++---
.../external/parser/AbstractNestedDataParser.java | 2 +-
.../external/parser/DelimitedDataParser.java | 2 +-
.../asterix/external/parser/JSONDataParser.java | 437 ++-------------------
.../asterix/external/parser/ParseException.java | 4 +
.../asterix/external/parser/TweetParser.java | 2 +-
.../{ObjectPool.java => AbstractObjectPool.java} | 28 +-
.../external/parser/jackson/IObjectPool.java | 13 +-
.../external/parser/jackson/ObjectPool.java | 34 +-
.../external/parser/jackson/ParserContext.java | 8 +-
.../{ObjectPool.java => SoftObjectPool.java} | 38 +-
.../asterix/external/util/ExternalDataUtils.java | 64 ++-
.../asterix/external/library/TypeNameFactory.java | 14 +-
.../asterix/external/library/TypeNameFunction.java | 30 +-
.../asterix/metadata/entities/NodeGroup.java | 18 +-
.../functions/ExternalFunctionCompilerUtil.java | 2 +-
.../metadata/functions/ExternalTypeComputer.java | 13 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../utils/SecondaryIndexOperationsHelper.java | 4 +
.../asterix/om/functions/BuiltinFunctions.java | 3 -
.../org/apache/asterix/om/types/BuiltinType.java | 60 +--
.../replication/management/NetworkingUtil.java | 34 +-
.../DecodeDataverseDisplayNameDescriptor.java | 88 -----
.../runtime/functions/FunctionCollection.java | 2 -
.../asterix/runtime/utils/ClusterStateManager.java | 2 -
asterixdb/asterix-server/pom.xml | 4 +-
.../service/locking/ResourceGroupTable.java | 2 +-
asterixdb/pom.xml | 4 +-
.../appended-resources/supplemental-models.xml | 8 +-
....com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
.../LogicalOperatorPrettyPrintVisitorJson.java | 9 +-
.../rules/RemoveRedundantVariablesRule.java | 94 +++--
.../api/rewriter/runtime/SuperActivity.java | 5 +
.../runtime/SuperActivityOperatorNodePushable.java | 7 +-
.../apache/hyracks/api/util/ErrorMessageUtil.java | 11 +-
.../hyracks/api/util/HyracksThrowingSupplier.java} | 8 +-
.../org/apache/hyracks/api/util/InvokeUtil.java | 3 +
.../java/org/apache/hyracks}/util/ParseUtil.java | 25 +-
.../src/main/resources/errormsg/en.properties | 2 +-
.../apache/hyracks/control/cc/job/JobManager.java | 4 +-
.../file/FieldCursorForDelimitedDataParser.java | 5 +-
.../hyracks/http/server/AbstractServlet.java | 23 +-
.../hyracks/http/server/HttpServerHandler.java | 8 +-
.../apache/hyracks/http/server/utils/HttpUtil.java | 31 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 6 +-
.../util/MaintainedThreadNameExecutorService.java | 12 +-
.../java/org/apache/hyracks/util/NetworkUtil.java | 22 ++
.../java/org/apache/hyracks/util/StorageUtil.java | 53 +--
.../org/apache/hyracks/util/ThrowingFunction.java | 19 +
.../apache/hyracks/util/string/UTF8StringUtil.java | 11 +-
.../hyracks/util/string/UTF8StringWriter.java | 3 +-
.../org/apache/hyracks/util/NetworkUtilTest.java | 11 +
.../org/apache/hyracks/util/StorageUnitTest.java | 79 ++++
hyracks-fullstack/pom.xml | 18 +-
137 files changed, 1095 insertions(+), 1098 deletions(-)
diff --cc asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index 17159d3,e1c9164..18d55aa
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@@ -331,11 -331,10 +331,10 @@@ import org.apache.asterix.runtime.evalu
import org.apache.asterix.runtime.evaluators.functions.CreateQueryUIDDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateRectangleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor;
- import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseDisplayNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.FullTextContainsWithoutOptionDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.FullTextContainsFunctionDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.FullTextContainsWithoutOptionFunctionDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetItemDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetJobParameterByNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetTypeDescriptor;
[asterixdb] 11/38: [NO ISSUE][OTH] Add request parameter to allow
for immediate execution
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0da78d6f840836af98004ad129fcb10b6d166e72
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Apr 22 18:29:22 2021 +0300
[NO ISSUE][OTH] Add request parameter to allow for immediate execution
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Add a request parameter to allow certain queries to
run immediately.
Change-Id: I60208c07200326a4957e908d8e7c9dcdb7bc3204
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11145
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/translator/IRequestParameters.java | 2 ++
.../asterix/translator/IStatementExecutor.java | 6 +++++-
.../apache/asterix/api/common/APIFramework.java | 24 +++++++++++++---------
.../message/ExecuteStatementRequestMessage.java | 20 +++++++++++++++---
.../asterix/app/translator/QueryTranslator.java | 16 +++++++--------
.../asterix/app/translator/RequestParameters.java | 17 +++++++++++++++
.../org/apache/asterix/utils/FeedOperations.java | 2 +-
7 files changed, 64 insertions(+), 23 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 4ad1040..5c743ee 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -73,4 +73,6 @@ public interface IRequestParameters extends ICommonRequestParameters {
* inconsistent.
*/
boolean isForceDropDataset();
+
+ boolean isSkipAdmissionPolicy();
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 083fa83..d60b791 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -273,6 +273,9 @@ public interface IStatementExecutor {
* @param statementParameters
* Statement parameters
* @param statementRewriter
+ * The statement rewriter
+ * @param requestParameters
+ * The request parameters
* @return the compiled {@code JobSpecification}
* @throws AsterixException
* @throws RemoteException
@@ -281,7 +284,8 @@ public interface IStatementExecutor {
*/
JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
Query query, ICompiledDmlStatement dmlStatement, Map<String, IAObject> statementParameters,
- IStatementRewriter statementRewriter) throws RemoteException, AlgebricksException, ACIDException;
+ IStatementRewriter statementRewriter, IRequestParameters requestParameters)
+ throws RemoteException, AlgebricksException, ACIDException;
/**
* returns the active dataverse for an entity or a statement
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index b3c7f54..920b0d9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -77,6 +77,7 @@ import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.SessionOutput;
import org.apache.asterix.translator.SqlppExpressionToPlanTranslator;
@@ -205,7 +206,8 @@ public class APIFramework {
public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
Query query, int varCounter, String outputDatasetName, SessionOutput output,
ICompiledDmlStatement statement, Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer,
- IWarningCollector warningCollector) throws AlgebricksException, ACIDException {
+ IWarningCollector warningCollector, IRequestParameters requestParameters)
+ throws AlgebricksException, ACIDException {
// establish facts
final boolean isQuery = query != null;
@@ -307,15 +309,17 @@ public class APIFramework {
JobSpecification spec = compiler.createJob(metadataProvider.getApplicationContext(), jobEventListenerFactory);
if (isQuery) {
- // Sets a required capacity, only for read-only queries.
- // DDLs and DMLs are considered not that frequent.
- // limit the computation locations to the locations that will be used in the query
- final INodeJobTracker nodeJobTracker = metadataProvider.getApplicationContext().getNodeJobTracker();
- final AlgebricksAbsolutePartitionConstraint jobLocations =
- getJobLocations(spec, nodeJobTracker, computationLocations);
- final IClusterCapacity jobRequiredCapacity =
- ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
- spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) {
+ // Sets a required capacity, only for read-only queries.
+ // DDLs and DMLs are considered not that frequent.
+ // limit the computation locations to the locations that will be used in the query
+ final INodeJobTracker nodeJobTracker = metadataProvider.getApplicationContext().getNodeJobTracker();
+ final AlgebricksAbsolutePartitionConstraint jobLocations =
+ getJobLocations(spec, nodeJobTracker, computationLocations);
+ final IClusterCapacity jobRequiredCapacity =
+ ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
+ spec.setRequiredClusterCapacity(jobRequiredCapacity);
+ }
}
if (isQuery && conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
generateJob(spec);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index d3345fb..29ee76d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -87,12 +87,24 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
private final ProfileType profileType;
private final IRequestReference requestReference;
private final boolean forceDropDataset;
+ private final boolean skipAdmissionPolicy;
public ExecuteStatementRequestMessage(String requestNodeId, long requestMessageId, ILangExtension.Language lang,
String statementsText, SessionConfig sessionConfig, ResultProperties resultProperties,
String clientContextID, String handleUrl, Map<String, String> optionalParameters,
Map<String, byte[]> statementParameters, boolean multiStatement, ProfileType profileType,
int statementCategoryRestrictionMask, IRequestReference requestReference, boolean forceDropDataset) {
+ this(requestNodeId, requestMessageId, lang, statementsText, sessionConfig, resultProperties, clientContextID,
+ handleUrl, optionalParameters, statementParameters, multiStatement, profileType,
+ statementCategoryRestrictionMask, requestReference, forceDropDataset, false);
+ }
+
+ protected ExecuteStatementRequestMessage(String requestNodeId, long requestMessageId, ILangExtension.Language lang,
+ String statementsText, SessionConfig sessionConfig, ResultProperties resultProperties,
+ String clientContextID, String handleUrl, Map<String, String> optionalParameters,
+ Map<String, byte[]> statementParameters, boolean multiStatement, ProfileType profileType,
+ int statementCategoryRestrictionMask, IRequestReference requestReference, boolean forceDropDataset,
+ boolean skipAdmissionPolicy) {
this.requestNodeId = requestNodeId;
this.requestMessageId = requestMessageId;
this.lang = lang;
@@ -108,6 +120,7 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
this.profileType = profileType;
this.requestReference = requestReference;
this.forceDropDataset = forceDropDataset;
+ this.skipAdmissionPolicy = skipAdmissionPolicy;
}
@Override
@@ -150,9 +163,10 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
final IStatementExecutor.StatementProperties statementProperties =
new IStatementExecutor.StatementProperties();
Map<String, IAObject> stmtParams = RequestParameters.deserializeParameterValues(statementParameters);
- final IRequestParameters requestParameters = new RequestParameters(requestReference, statementsText, null,
- resultProperties, stats, statementProperties, outMetadata, clientContextID, optionalParameters,
- stmtParams, multiStatement, statementCategoryRestrictionMask, forceDropDataset);
+ final IRequestParameters requestParameters =
+ new RequestParameters(requestReference, statementsText, null, resultProperties, stats,
+ statementProperties, outMetadata, clientContextID, optionalParameters, stmtParams,
+ multiStatement, statementCategoryRestrictionMask, forceDropDataset, skipAdmissionPolicy);
translator.compileAndExecute(ccApp.getHcc(), requestParameters);
translator.getWarnings(warnings, maxWarnings - warnings.size());
stats.updateTotalWarningsCount(parserTotalWarningsCount);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index d49649d..754e2b6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2781,7 +2781,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
loadStmt.getDatasetName(), loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted());
cls.setSourceLocation(stmt.getSourceLocation());
JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls,
- null, responsePrinter, warningCollector);
+ null, responsePrinter, warningCollector, null);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
@@ -2880,8 +2880,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
CompiledDeleteStatement clfrqs = new CompiledDeleteStatement(stmtDelete.getVariableExpr(), dataverseName,
datasetName, stmtDelete.getCondition(), stmtDelete.getVarCounter(), stmtDelete.getQuery());
clfrqs.setSourceLocation(stmt.getSourceLocation());
- JobSpecification jobSpec =
- rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams, stmtRewriter);
+ JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams,
+ stmtRewriter, null);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -2904,7 +2904,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
@Override
public JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector,
MetadataProvider metadataProvider, Query query, ICompiledDmlStatement stmt,
- Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter)
+ Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter, IRequestParameters requestParameters)
throws AlgebricksException, ACIDException {
Map<VarIdentifier, IAObject> externalVars = createExternalVariables(stmtParams, stmtRewriter);
@@ -2916,7 +2916,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
// Query Compilation (happens under the same ongoing metadata transaction)
return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first,
rewrittenResult.second, stmt == null ? null : stmt.getDatasetName(), sessionOutput, stmt, externalVars,
- responsePrinter, warningCollector);
+ responsePrinter, warningCollector, requestParameters);
}
private JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector,
@@ -2955,7 +2955,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
// transaction)
return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, rewrittenInsertUpsert.getQuery(),
rewrittenResult.second, datasetName, sessionOutput, clfrqs, externalVars, responsePrinter,
- warningCollector);
+ warningCollector, null);
}
protected void handleCreateFeedStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception {
@@ -3419,8 +3419,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- final JobSpecification jobSpec =
- rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, stmtRewriter);
+ final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams,
+ stmtRewriter, requestParameters);
// update stats with count of compile-time warnings. needs to be adapted for multi-statement.
stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
afterCompile();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index d5ea685..6c8f21c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -59,6 +59,7 @@ public class RequestParameters implements IRequestParameters {
private final int statementCategoryRestrictionMask;
private final String statement;
private final boolean forceDropDataset;
+ private final boolean skipAdmissionPolicy;
public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
@@ -83,6 +84,16 @@ public class RequestParameters implements IRequestParameters {
IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
int statementCategoryRestrictionMask, boolean forceDropDataset) {
+ this(requestReference, statement, resultSet, resultProperties, stats, statementProperties, outMetadata,
+ clientContextId, optionalParameters, statementParameters, multiStatement,
+ statementCategoryRestrictionMask, forceDropDataset, false);
+ }
+
+ public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats, StatementProperties statementProperties,
+ IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
+ Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement,
+ int statementCategoryRestrictionMask, boolean forceDropDataset, boolean skipAdmissionPolicy) {
this.requestReference = requestReference;
this.statement = statement;
this.resultSet = resultSet;
@@ -96,6 +107,7 @@ public class RequestParameters implements IRequestParameters {
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask = statementCategoryRestrictionMask;
this.forceDropDataset = forceDropDataset;
+ this.skipAdmissionPolicy = skipAdmissionPolicy;
}
@Override
@@ -149,6 +161,11 @@ public class RequestParameters implements IRequestParameters {
}
@Override
+ public boolean isSkipAdmissionPolicy() {
+ return skipAdmissionPolicy;
+ }
+
+ @Override
public Map<String, IAObject> getStatementParameters() {
return statementParameters;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index dcd52a0..9472da5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -253,7 +253,7 @@ public class FeedOperations {
clfrqs = new CompiledStatements.CompiledUpsertStatement(feedConn.getDataverseName(),
feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null);
}
- return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null);
+ return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null, null);
}
private static JobSpecification combineIntakeCollectJobs(MetadataProvider metadataProvider, Feed feed,
[asterixdb] 37/38: [NO ISSUE][HYR][HTTP] http stream handler ->
consumer / processor
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 34351729058b19a530b9b4aa193433141840c352
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 19 08:00:00 2021 -0400
[NO ISSUE][HYR][HTTP] http stream handler -> consumer / processor
Change-Id: I7cd7622dbee880845d0b4233ce3a3b17af15eebc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11524
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../org/apache/hyracks/http/server/utils/HttpUtil.java | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index c61e0d2..835cd54 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -45,6 +45,7 @@ import org.apache.hyracks.http.server.BaseRequest;
import org.apache.hyracks.http.server.FormUrlEncodedRequest;
import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.util.ThrowingConsumer;
+import org.apache.hyracks.util.ThrowingFunction;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -191,16 +192,23 @@ public class HttpUtil {
return i < 0 ? uri : uri.substring(0, i);
}
- public static void handleStreamInterruptibly(CloseableHttpResponse response,
+ public static void consumeStreamInterruptibly(CloseableHttpResponse response,
ThrowingConsumer<Reader> streamProcessor, ExecutorService executor, Supplier<String> descriptionSupplier)
+ throws InterruptedException, ExecutionException, IOException {
+ processStreamInterruptibly(response, ThrowingConsumer.asFunction(streamProcessor), executor,
+ descriptionSupplier);
+ }
+
+ public static <T> T processStreamInterruptibly(CloseableHttpResponse response,
+ ThrowingFunction<Reader, T> streamProcessor, ExecutorService executor, Supplier<String> descriptionSupplier)
throws IOException, InterruptedException, ExecutionException {
// we have to consume the stream in a separate thread, as it not stop on interrupt; we need to
// instead close the connection to achieve the interrupt
String description = descriptionSupplier.get();
- Future<Void> readFuture = executor.submit(() -> {
+ Future<T> readFuture = executor.submit(() -> {
Thread.currentThread().setName(description);
InputStreamReader reader = new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8);
- streamProcessor.process(new Reader() {
+ return streamProcessor.process(new Reader() {
@Override
public int read(char[] cbuf, int off, int len) throws IOException {
return reader.read(cbuf, off, len);
@@ -213,10 +221,9 @@ public class HttpUtil {
LOGGER.debug("ignoring close on {}", reader);
}
});
- return null;
});
try {
- readFuture.get();
+ return readFuture.get();
} catch (InterruptedException ex) { // NOSONAR -- interrupt or rethrow
response.close();
try {
[asterixdb] 23/38: [ASTERIXDB-2897][FUN] Fix unknown handling in
PyUDF
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ec5195a4f58b2ce3971b537b48078f3bc696e0bf
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue May 4 09:35:21 2021 -0700
[ASTERIXDB-2897][FUN] Fix unknown handling in PyUDF
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Carry the fix from ASTERIXDB-2883 to Python UDFs,
and add a corresponding test for both this new
code in the non-batched execution, as well as
checking that it works the same in batched mode
Change-Id: I64ac16d094caafcc657a21650011ed41d0c3c6d6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11324
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
...3.query.sqlpp => type_validation.10.query.sqlpp} | 7 +++++--
...ion.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} | 0
.../type_validation.2.ddl.sqlpp | 4 +++-
.../type_validation.3.query.sqlpp | 3 ++-
....3.query.sqlpp => type_validation.4.query.sqlpp} | 5 ++++-
....3.query.sqlpp => type_validation.5.query.sqlpp} | 5 +++--
....3.query.sqlpp => type_validation.6.query.sqlpp} | 7 +++++--
....3.query.sqlpp => type_validation.7.query.sqlpp} | 3 ++-
....3.query.sqlpp => type_validation.8.query.sqlpp} | 5 ++++-
....3.query.sqlpp => type_validation.9.query.sqlpp} | 5 +++--
.../type_validation.1.adm | 2 +-
.../type_validation.2.adm | 1 +
.../type_validation.3.adm | 1 +
.../type_validation.4.adm | 1 +
.../type_validation.5.adm | 1 +
.../type_validation.6.adm | 1 +
.../type_validation.7.adm | 1 +
.../type_validation.8.adm | 1 +
.../ExternalScalarPythonFunctionEvaluator.java | 21 ++++++++++++++++++---
19 files changed, 57 insertions(+), 17 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
index 0ae7d0c..69a0b25 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
index a8ba8a1..8041b5e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
@@ -18,6 +18,8 @@
*/
use externallibtest;
-create function typeValidation(a, b, c, d, e, f, g)
+create function typeValidation(a, b, c, d, e, f, g, h)
as "roundtrip", "Tests.roundtrip" at testlib;
+create function typeValidationNullCall(a, b, c, d, e, f, g, h)
+ as "roundtrip", "Tests.roundtrip" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
index 0ae7d0c..f9aaf57 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
index 0ae7d0c..884f31f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
index 0ae7d0c..fdd71ef 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
index 0ae7d0c..b85e0d0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
index 0ae7d0c..fa76f1a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
index 0ae7d0c..30640fd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
index 0ae7d0c..810ba3c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
index 93f8aec..19765bd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
@@ -1 +1 @@
-[ 907, 9.07, "907", 9.07, true, 15706, 621173594 ]
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
new file mode 100644
index 0000000..19765bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
@@ -0,0 +1 @@
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index e664f47..eb87399 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.library;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import static org.msgpack.core.MessagePack.Code.FIXARRAY_PREFIX;
import java.io.DataOutput;
@@ -88,17 +89,31 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
argHolder.clear();
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
for (int i = 0, ln = argEvals.length; i < ln; i++) {
argEvals[i].evaluate(tuple, argValues[i]);
- if (!finfo.getNullCall() && PointableHelper.checkAndSetMissingOrNull(result, argValues[i])) {
- return;
+ if (!nullCall) {
+ byte[] argBytes = argValues[i].getByteArray();
+ int argStart = argValues[i].getStartOffset();
+ ATypeTag argType = ATYPETAGDESERIALIZER.deserialize(argBytes[argStart]);
+ if (argType == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (argType == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
}
try {
- PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, finfo.getNullCall());
+ PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, nullCall);
} catch (IOException e) {
throw new HyracksDataException("Error evaluating Python UDF", e);
}
}
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ return;
+ }
try {
ByteBuffer res = libraryEvaluator.callPython(fnId, argHolder, argTypes.length);
resultBuffer.reset();
[asterixdb] 02/38: [NO ISSUE][REP] Account for SSL socket data
transfer behavior
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 94b1306e380dc923a964abf60900d55e545a4e4f
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Thu Apr 15 02:53:08 2021 +0300
[NO ISSUE][REP] Account for SSL socket data transfer behavior
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Account for the fact that a call to an SSLSocketChannel
read operation can return 0 read bytes even in a blocking
socket due to incomplete SSL packet.
Change-Id: I0e1e69cba7336e0cfca5def870ab16334ce8c19f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11064
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../replication/management/NetworkingUtil.java | 34 ++++++++++++----------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index 97dd049..7f6439c 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -60,30 +60,32 @@ public class NetworkingUtil {
public static void sendFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
long pos = 0;
- long fileSize = fileChannel.size();
- long remainingBytes = fileSize;
- long transferredBytes = 0;
-
- while ((transferredBytes += fileChannel.transferTo(pos, remainingBytes, socketChannel)) < fileSize) {
- pos += transferredBytes;
- remainingBytes -= transferredBytes;
+ long remainingBytes = fileChannel.size();
+ try {
+ while (remainingBytes > 0) {
+ long sentBytes = fileChannel.transferTo(pos, remainingBytes, socketChannel);
+ pos += sentBytes;
+ remainingBytes -= sentBytes;
+ }
+ socketChannel.getSocketChannel().socket().getOutputStream().flush();
+ } catch (Exception e) {
+ LOGGER.info("failed to send file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
}
- socketChannel.getSocketChannel().socket().getOutputStream().flush();
}
public static void downloadFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
+ long remainingBytes = fileChannel.size();
long pos = 0;
- long fileSize = fileChannel.size();
- long count = fileSize;
- long numTransferred = 0;
try {
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ while (remainingBytes > 0) {
+ long readBytes = fileChannel.transferFrom(socketChannel, pos, remainingBytes);
+ pos += readBytes;
+ remainingBytes -= readBytes;
}
} catch (Exception e) {
- LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
- count, numTransferred);
+ LOGGER.info("failed to download file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
throw e;
}
}
[asterixdb] 17/38: Merge tracking branch
'gerrit/stabilization-f69489'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2fe0e14266c32fbb2379b859acb08d095f109255
Merge: f9f467a 3856cb3
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat May 1 15:00:00 2021 -0400
Merge tracking branch 'gerrit/stabilization-f69489'
Change-Id: Ide3d9da3f2c2518afd0b488f6db5b2e3377ee8a7
asterixdb/asterix-server/pom.xml | 4 ++--
asterixdb/pom.xml | 4 ++--
.../main/appended-resources/supplemental-models.xml | 8 ++++----
...tent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
hyracks-fullstack/pom.xml | 18 +++++++++---------
5 files changed, 17 insertions(+), 17 deletions(-)
diff --cc asterixdb/pom.xml
index f57d71e,ed486fd..36b4581
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -73,15 -71,13 +73,15 @@@
<global.itest.excludes/>
<itest.includes>${global.itest.includes}</itest.includes>
<itest.excludes>${global.itest.excludes}</itest.excludes>
+ <license.stage>compile</license.stage>
+ <resource.stage>process-classes</resource.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <algebricks.version>0.3.4.2-SNAPSHOT</algebricks.version>
- <hyracks.version>0.3.4.2-SNAPSHOT</hyracks.version>
+ <algebricks.version>0.3.5-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.3.5-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
[asterixdb] 04/38: [ASTERIXDB-2874][EXT] S3 external datasets: add
support to anonymous authentication
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c70687f57f1bf93727b746a6bb787e374dfec0f4
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Thu Apr 15 17:47:31 2021 +0300
[ASTERIXDB-2874][EXT] S3 external datasets: add support to anonymous authentication
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added support to creating external datasets with no
credentials (anonymous).
Change-Id: I9b0b83ac51b4c262b52d657f5cdcf6c0f818451e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11023
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../s3/anonymous_no_auth/test.000.ddl.sqlpp | 36 +++++++++++++++++
.../s3/anonymous_no_auth/test.001.ddl.sqlpp | 36 +++++++++++++++++
.../s3/anonymous_no_auth/test.002.ddl.sqlpp | 35 +++++++++++++++++
.../s3/anonymous_no_auth/test.099.ddl.sqlpp | 20 ++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 7 ++++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../asterix/external/util/ExternalDataUtils.java | 45 +++++++++++++++++-----
8 files changed, 172 insertions(+), 9 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
new file mode 100644
index 0000000..13cfe8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("accessKeyId"="dummyAccessKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
new file mode 100644
index 0000000..b8d0945
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
new file mode 100644
index 0000000..9eda057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 6557230..9bc463c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -148,6 +148,13 @@
<output-dir compare="Text">non-s3-region</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="anonymous_no_auth">
+ <output-dir compare="Text">anonymous_no_auth</output-dir>
+ <expected-error>ASX3119: Parameter 'secretAccessKey' is required if 'accessKeyId' is provided</expected-error>
+ <expected-error>ASX3119: Parameter 'accessKeyId' is required if 'secretAccessKey' is provided</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common">
<compilation-unit name="query-with-limit-plan">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 210ce0f..7e32e0b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -348,6 +348,7 @@ public enum ErrorCode implements IError {
INPUT_DECODE_FAILURE(3116),
FAILED_TO_PARSE_MALFORMED_LOG_RECORD(3117),
ACTIVE_ENTITY_NOT_RUNNING(3118),
+ REQUIRED_PARAM_IF_PARAM_IS_PRESENT(3119),
// Lifecycle management errors
DUPLICATE_PARTITION_ID(4000),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 93fa828..345b9d8 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -353,6 +353,7 @@
3116 = Failed to decode input
3117 = Failed to parse record, malformed log record
3118 = Active Entity %1$s is not running (it is %2$s)
+3119 = Parameter '%1$s' is required if '%2$s' is provided
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 2c29e60..f5c62a5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,7 +18,10 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
@@ -84,8 +87,9 @@ import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.ListBlobsOptions;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkException;
@@ -726,8 +730,8 @@ public class ExternalDataUtils {
*/
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
@@ -735,14 +739,23 @@ public class ExternalDataUtils {
S3ClientBuilder builder = S3Client.builder();
// Credentials
- AwsCredentials credentials;
- if (sessionToken != null) {
- credentials = AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken);
+ AwsCredentialsProvider credentialsProvider;
+
+ // No auth required
+ if (accessKeyId == null) {
+ credentialsProvider = AnonymousCredentialsProvider.create();
} else {
- credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey);
+ // auth required, check for temporary or permanent credentials
+ if (sessionToken != null) {
+ credentialsProvider = StaticCredentialsProvider
+ .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+ } else {
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ }
}
- builder.credentialsProvider(StaticCredentialsProvider.create(credentials));
+ builder.credentialsProvider(credentialsProvider);
builder.region(Region.of(regionId));
// Validate the service endpoint if present
@@ -777,10 +790,24 @@ public class ExternalDataUtils {
throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
}
+ // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ if (accessKeyId == null || secretAccessKey == null) {
+ // If one is passed, the other is required
+ if (accessKeyId != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ } else if (secretAccessKey != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ }
+
validateIncludeExclude(configuration);
// Check if the bucket is present
- S3Client s3Client = buildAwsS3Client(configuration);;
+ S3Client s3Client = buildAwsS3Client(configuration);
S3Response response;
boolean useOldApi = false;
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
[asterixdb] 14/38: [NO ISSUE][FUN] Remove
decode_dataverse_display_name()
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 61aad744bcb446f976bc79a9feedce88256bbc26
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Apr 27 12:19:43 2021 -0700
[NO ISSUE][FUN] Remove decode_dataverse_display_name()
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Remove decode_dataverse_display_name() function
Change-Id: Iae38424c92ba237b58032958b67a6e2026615f0d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11203
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../special_chars_2/special_chars_2.2.query.sqlpp | 1 -
.../special_chars_2/special_chars_2.2.adm | 12 +--
.../asterix/om/functions/BuiltinFunctions.java | 3 -
.../DecodeDataverseDisplayNameDescriptor.java | 88 ----------------------
.../runtime/functions/FunctionCollection.java | 2 -
5 files changed, 6 insertions(+), 100 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
index 4dc4e3b..08a41b5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/multipart-dataverse/special_chars_2/special_chars_2.2.query.sqlpp
@@ -18,7 +18,6 @@
*/
select DataverseName as CanonicalName,
- decode_dataverse_display_name(DataverseName) as DisplayName,
decode_dataverse_name(DataverseName) as NameParts
from Metadata.`Dataverse`
order by DataverseName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
index 7edd3eb..805247c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/multipart-dataverse/special_chars_2/special_chars_2.2.adm
@@ -1,6 +1,6 @@
-{ "CanonicalName": "A", "DisplayName": "A", "NameParts": [ "A" ] }
-{ "CanonicalName": "B/C", "DisplayName": "B.C", "NameParts": [ "B", "C" ] }
-{ "CanonicalName": "C.D.E", "DisplayName": "`C.D.E`", "NameParts": [ "C.D.E" ] }
-{ "CanonicalName": "Default", "DisplayName": "Default", "NameParts": [ "Default" ] }
-{ "CanonicalName": "Metadata", "DisplayName": "Metadata", "NameParts": [ "Metadata" ] }
-{ "CanonicalName": "a-A/b_B/c$C/z.Z", "DisplayName": "`a-A`.b_B.c$C.`z.Z`", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
+{ "CanonicalName": "A", "NameParts": [ "A" ] }
+{ "CanonicalName": "B/C", "NameParts": [ "B", "C" ] }
+{ "CanonicalName": "C.D.E", "NameParts": [ "C.D.E" ] }
+{ "CanonicalName": "Default", "NameParts": [ "Default" ] }
+{ "CanonicalName": "Metadata", "NameParts": [ "Metadata" ] }
+{ "CanonicalName": "a-A/b_B/c$C/z.Z", "NameParts": [ "a-A", "b_B", "c$C", "z.Z" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index aac01f9..a2e235e 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1592,8 +1592,6 @@ public class BuiltinFunctions {
public static final FunctionIdentifier DECODE_DATAVERSE_NAME =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "decode-dataverse-name", 1);
- public static final FunctionIdentifier DECODE_DATAVERSE_DISPLAY_NAME =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "decode-dataverse-display-name", 1);
static {
// first, take care of Algebricks builtin functions
@@ -2366,7 +2364,6 @@ public class BuiltinFunctions {
addPrivateFunction(META_KEY, AnyTypeComputer.INSTANCE, false);
addFunction(DECODE_DATAVERSE_NAME, OrderedListOfAStringTypeComputer.INSTANCE_NULLABLE, true);
- addFunction(DECODE_DATAVERSE_DISPLAY_NAME, AStringTypeComputer.INSTANCE_NULLABLE, true);
addPrivateFunction(COLLECTION_TO_SEQUENCE, CollectionToSequenceTypeComputer.INSTANCE, true);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java
deleted file mode 100644
index fd78b72..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DecodeDataverseDisplayNameDescriptor.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.IOException;
-
-import org.apache.asterix.common.annotations.MissingNullInOutFunction;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-
-@MissingNullInOutFunction
-public class DecodeDataverseDisplayNameDescriptor extends AbstractScalarFunctionDynamicDescriptor {
- private static final long serialVersionUID = 1L;
-
- public static final IFunctionDescriptorFactory FACTORY = DecodeDataverseDisplayNameDescriptor::new;
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
- return new AbstractUnaryStringStringEval(ctx, args[0], getIdentifier(), sourceLoc) {
-
- private final StringBuilder sb = new StringBuilder();
-
- @Override
- void process(UTF8StringPointable inputString, IPointable resultPointable) throws IOException {
- String dataverseCanonicalName = inputString.toString();
-
- sb.setLength(0);
- try {
- DataverseName.getDisplayFormFromCanonicalForm(dataverseCanonicalName, sb);
- } catch (AsterixException e) {
- return; // writeResult() will emit NULL
- }
-
- resultBuilder.reset(resultArray, inputString.getUTF8Length());
- resultBuilder.appendString(sb);
- resultBuilder.finish();
- }
-
- @Override
- void writeResult(IPointable resultPointable) throws IOException {
- if (sb.length() == 0) {
- PointableHelper.setNull(resultPointable);
- } else {
- super.writeResult(resultPointable);
- }
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.DECODE_DATAVERSE_DISPLAY_NAME;
- }
-}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index 42e27da..e1c9164 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -331,7 +331,6 @@ import org.apache.asterix.runtime.evaluators.functions.CreatePolygonDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateQueryUIDDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateRectangleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseDisplayNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescriptor;
@@ -1215,7 +1214,6 @@ public final class FunctionCollection implements IFunctionCollection {
// Other functions
fc.add(DecodeDataverseNameDescriptor.FACTORY);
- fc.add(DecodeDataverseDisplayNameDescriptor.FACTORY);
fc.add(RandomWithSeedDescriptor.FACTORY);
ServiceLoader.load(IFunctionRegistrant.class).iterator().forEachRemaining(c -> c.register(fc));
[asterixdb] 33/38: Merge branch 'gerrit/mad-hatter'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2782be4244396a3dbc4ddfd1dc58ffd27f786e8c
Merge: dec8426 d5de9eb
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 19 07:35:35 2021 -0400
Merge branch 'gerrit/mad-hatter'
Change-Id: I78240c264b37e3d5be3fc40ed1f2b50f20c8689c
.../java/org/apache/hyracks/util/StorageUtil.java | 53 ++++++---------
.../org/apache/hyracks/util/StorageUnitTest.java | 79 ++++++++++++++++++++++
2 files changed, 99 insertions(+), 33 deletions(-)
[asterixdb] 29/38: [NO ISSUE][MISC] Fix typo on active state log,
+= HyracksThrowingSupplier
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b69e30e1c0b55d6216647f28e3b68a16a66028e0
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 12 12:06:33 2021 -0400
[NO ISSUE][MISC] Fix typo on active state log, += HyracksThrowingSupplier
Change-Id: I19ba60f6a362b06451be331449d21f3c37653570
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11443
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ian Maxon <im...@uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../app/active/ActiveEntityEventsListener.java | 7 ++----
.../hyracks/api/util/HyracksThrowingSupplier.java | 26 ++++++++++++++++++++++
2 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 2891710..e18e50c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -133,15 +133,12 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl
this.locations = locations;
this.numRegistered = 0;
this.numDeRegistered = 0;
- this.handler =
- (ActiveNotificationHandler) metadataProvider.getApplicationContext().getActiveNotificationHandler();
+ this.handler = (ActiveNotificationHandler) appCtx.getActiveNotificationHandler();
handler.registerListener(this);
}
protected synchronized void setState(ActivityState newState) {
- if (LOGGER.isEnabled(level)) {
- LOGGER.log(level, "State of " + getEntityId() + "is being set to " + newState + " from " + state);
- }
+ LOGGER.log(level, "State of {} is being set to {} from {}", getEntityId(), newState, state);
this.prevState = state;
this.state = newState;
if (newState == ActivityState.STARTING || newState == ActivityState.RECOVERING
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java
new file mode 100644
index 0000000..c6c3922
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingSupplier.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@FunctionalInterface
+public interface HyracksThrowingSupplier<T> {
+ T get() throws HyracksDataException;
+}
[asterixdb] 36/38: [NO ISSUE][MISC] Improve parser error reporting
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3b6982ce7fa50ff4005d50847a944ecbaf3ecb30
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu May 20 16:43:43 2021 -0400
[NO ISSUE][MISC] Improve parser error reporting
Change-Id: I8707d9e2a952693b6501e9e63aff9162a77541cc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11543
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
...stsuite_external_dataset_azure_blob_storage.xml | 10 ++++-----
.../runtimets/testsuite_external_dataset_s3.xml | 10 ++++-----
...testsuite_external_dataset_s3_one_partition.xml | 20 ++++++++---------
.../reader/stream/QuotedLineRecordReader.java | 2 +-
.../reader/stream/SemiStructuredRecordReader.java | 2 +-
.../external/parser/AbstractJsonDataParser.java | 18 +++++++---------
.../external/parser/DelimitedDataParser.java | 2 +-
.../java/org/apache/hyracks}/util/ParseUtil.java | 25 +++++++++++++++++++---
.../src/main/resources/errormsg/en.properties | 2 +-
.../file/FieldCursorForDelimitedDataParser.java | 5 ++---
10 files changed, 56 insertions(+), 40 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index df60e60..1e302da 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -143,11 +143,11 @@
<compilation-unit name="common/malformed-json">
<placeholder name="adapter" value="AZUREBLOB" />
<output-dir compare="Text">common/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field 'field': Duplicate field 'field'</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field 'field': Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field 'field2': Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 9bc463c..b354e65 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -92,11 +92,11 @@
<compilation-unit name="common/malformed-json">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">common/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field 'field': Duplicate field 'field'</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field 'field': Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field 'field2': Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
+ <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field 'array_f': Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
index 92b5e32..d02647d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3_one_partition.xml
@@ -49,17 +49,17 @@
<output-dir compare="Text">common/csv-warnings</output-dir>
<expected-warn>Parsing error at data_dir/no_h_missing_fields.csv line 2 field 3: some fields are missing</expected-warn>
<expected-warn>Parsing error at data_dir/no_h_no_closing_q.csv line 2 field 0: malformed input record ended abruptly</expected-warn>
- <expected-warn>Parsing error at line 2 field 0: malformed input record ended abruptly</expected-warn>
+ <expected-warn>Parsing error at line 2 field 0: malformed input record ended abruptly</expected-warn>
- <expected-warn>Parsing error at line 5 field 3: invalid value</expected-warn>
- <expected-warn>Parsing error at line 2 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 11 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 3 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 4 field 1: invalid value</expected-warn>
- <expected-warn>Parsing error at line 7 field 7: invalid value</expected-warn>
- <expected-warn>Parsing error at line 13 field 7: invalid value</expected-warn>
- <expected-warn>Parsing error at line 12 field 3: invalid value</expected-warn>
- <expected-warn>Parsing error at line 9 field 6: a quote should be in the beginning</expected-warn>
+ <expected-warn>Parsing error at line 5 field 3: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 2 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 11 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 3 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 4 field 1: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 7 field 7: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 13 field 7: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 12 field 3: invalid value</expected-warn>
+ <expected-warn>Parsing error at line 9 field 6: a quote should be in the beginning</expected-warn>
<expected-warn>Parsing error at data_dir/h_invalid_values.csv line 5 field 3: invalid value</expected-warn>
<expected-warn>Parsing error at data_dir/h_invalid_values.csv line 2 field 1: invalid value</expected-warn>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
index 3a502d0..4c253bc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
@@ -29,10 +29,10 @@ import java.util.Map;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.util.ParseUtil;
public class QuotedLineRecordReader extends LineRecordReader {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
index 2ff5cfa..0e23e46 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/SemiStructuredRecordReader.java
@@ -40,10 +40,10 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.util.ParseUtil;
public class SemiStructuredRecordReader extends StreamRecordReader {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
index 2bf0df4..2d20cc2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.external.parser;
+import static org.apache.hyracks.api.exceptions.ErrorCode.PARSING_ERROR;
+
import java.io.DataOutput;
import java.io.IOException;
import java.util.BitSet;
@@ -46,6 +48,7 @@ import org.apache.asterix.runtime.exceptions.UnsupportedTypeException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.util.ParseUtil;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
@@ -433,18 +436,13 @@ public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<AD
}
long lineNum = lineNumber.getAsLong() + jsonParser.getCurrentLocation().getLineNr() - 1;
JsonStreamContext parsingContext = jsonParser.getParsingContext();
- String fieldName = "N/A";
- while (parsingContext != null) {
- String currentFieldName = parsingContext.getCurrentName();
- if (currentFieldName != null) {
- fieldName = currentFieldName;
- break;
- }
+ String fieldName = null;
+ while (parsingContext != null && fieldName == null) {
+ fieldName = parsingContext.getCurrentName();
parsingContext = parsingContext.getParent();
}
-
- return HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.PARSING_ERROR,
- dataSourceName.get(), lineNum, fieldName, msg);
+ final String locationDetails = ParseUtil.asLocationDetailString(dataSourceName.get(), lineNum, fieldName);
+ return HyracksDataException.create(PARSING_ERROR, locationDetails, msg);
}
return new RuntimeDataException(ErrorCode.RECORD_READER_MALFORMED_INPUT_STREAM, e);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
index 60e6e77..590f51d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java
@@ -38,7 +38,6 @@ import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.api.IRecordDataParser;
import org.apache.asterix.external.api.IStreamDataParser;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ParseUtil;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
@@ -52,6 +51,7 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParser;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.std.file.FieldCursorForDelimitedDataParser;
+import org.apache.hyracks.util.ParseUtil;
public class DelimitedDataParser extends AbstractDataParser implements IStreamDataParser, IRecordDataParser<char[]> {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java
similarity index 56%
rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java
index 5a46af7..63fec09 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/ParseUtil.java
@@ -16,7 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+package org.apache.hyracks.util;
+
+import java.util.StringJoiner;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -29,7 +31,24 @@ public class ParseUtil {
public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
String warnMessage) {
- warningCollector
- .warn(Warning.of(null, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
+ warningCollector.warn(Warning.of(null, ErrorCode.PARSING_ERROR,
+ asLocationDetailString(dataSourceName, lineNum, fieldNum), warnMessage));
+ }
+
+ public static String asLocationDetailString(String dataSource, long lineNum, Object fieldIdentifier) {
+ StringJoiner details = new StringJoiner(" ");
+ details.setEmptyValue("N/A");
+ if (dataSource != null && !dataSource.isEmpty()) {
+ details.add(dataSource);
+ }
+ if (lineNum >= 0) {
+ details.add("line " + lineNum);
+ }
+ if (fieldIdentifier instanceof Number) {
+ details.add("field " + fieldIdentifier);
+ } else if (fieldIdentifier instanceof String && !((String) fieldIdentifier).isEmpty()) {
+ details.add("field '" + fieldIdentifier + "'");
+ }
+ return "at " + details;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 9f04fb2..4d9c60b 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -141,7 +141,7 @@
121 = A numeric type promotion error has occurred: %1$s
122 = Encountered an error while printing the plan: %1$s
123 = Insufficient memory is provided for the join operators, please increase the join memory budget.
-124 = Parsing error at %1$s line %2$s field %3$s: %4$s
+124 = Parsing error %s: %s
125 = Invalid inverted list type traits: %1$s
126 = Illegal state. %1$s
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
index 936d63e..ffc87cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
@@ -23,9 +23,8 @@ import java.io.Reader;
import java.util.Arrays;
import java.util.function.Supplier;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.util.ParseUtil;
public class FieldCursorForDelimitedDataParser {
@@ -448,6 +447,6 @@ public class FieldCursorForDelimitedDataParser {
}
private void warn(String message) {
- warnings.warn(Warning.of(null, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount, message));
+ ParseUtil.warn(warnings, dataSourceName.get(), lineCount, fieldCount, message);
}
}
[asterixdb] 25/38: [NO ISSUE][*DB] Enable large parsing resources
to be freed on memory pressure
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5d02b03854911d7fb889b078f0ada37b282261f9
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu May 6 16:36:30 2021 -0400
[NO ISSUE][*DB] Enable large parsing resources to be freed on memory pressure
Change-Id: Ieaca566f765a5d3531baff8128d6ff03c95956e2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11365
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../{ObjectPool.java => AbstractObjectPool.java} | 28 +++++++---------
.../external/parser/jackson/IObjectPool.java | 25 +++++---------
.../external/parser/jackson/ObjectPool.java | 34 +++++--------------
.../external/parser/jackson/ParserContext.java | 8 ++---
.../{ObjectPool.java => SoftObjectPool.java} | 38 +++++++---------------
.../apache/hyracks/util/string/UTF8StringUtil.java | 11 ++++---
.../hyracks/util/string/UTF8StringWriter.java | 3 +-
7 files changed, 52 insertions(+), 95 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
similarity index 75%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
index 8945e71..ef75688 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
@@ -27,39 +27,35 @@ import org.apache.asterix.om.util.container.IObjectFactory;
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
+public abstract class AbstractObjectPool<E, T, Q> implements IObjectPool<E> {
private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
+ private final Queue<Q> recycledObjects;
+ private final T param;
- public ObjectPool() {
- this(null, null);
- }
-
- public ObjectPool(IObjectFactory<E, T> objectFactory) {
- this(objectFactory, null);
- }
-
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
+ protected AbstractObjectPool(IObjectFactory<E, T> objectFactory, T param) {
this.objectFactory = objectFactory;
recycledObjects = new ArrayDeque<>();
- this.element = element;
+ this.param = param;
}
public E getInstance() {
- E instance = recycledObjects.poll();
+ E instance = unwrap(recycledObjects.poll());
if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
+ instance = objectFactory.create(param);
}
return instance;
}
public void recycle(E object) {
if (object != null) {
- recycledObjects.add(object);
+ recycledObjects.add(wrap(object));
}
}
+ protected abstract E unwrap(Q element);
+
+ protected abstract Q wrap(E element);
+
@Override
public String toString() {
return recycledObjects.toString();
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
similarity index 57%
copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
index 563f865..a2990ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
@@ -16,23 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.util.string;
+package org.apache.asterix.external.parser.jackson;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-public class UTF8StringWriter implements Serializable {
-
- private static final long serialVersionUID = 1L;
- transient byte[] tempBytes;
-
- public final void writeUTF8(CharSequence str, DataOutput out) throws IOException {
- UTF8StringUtil.writeUTF8(str, out, this);
- }
-
- public final void writeUTF8(char[] buffer, int start, int length, DataOutput out) throws IOException {
- UTF8StringUtil.writeUTF8(buffer, start, length, out, this);
- }
+/**
+ * Object pool for DFS traversal mode, which allows to recycle objects
+ * as soon as it is not needed.
+ */
+public interface IObjectPool<E> {
+ E getInstance();
+ void recycle(E object);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
index 8945e71..864d9a9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
@@ -18,20 +18,13 @@
*/
package org.apache.asterix.external.parser.jackson;
-import java.util.ArrayDeque;
-import java.util.Queue;
-
import org.apache.asterix.om.util.container.IObjectFactory;
/**
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
- private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
-
+public class ObjectPool<E, T> extends AbstractObjectPool<E, T, E> {
public ObjectPool() {
this(null, null);
}
@@ -40,28 +33,17 @@ public class ObjectPool<E, T> {
this(objectFactory, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
- this.objectFactory = objectFactory;
- recycledObjects = new ArrayDeque<>();
- this.element = element;
+ public ObjectPool(IObjectFactory<E, T> objectFactory, T param) {
+ super(objectFactory, param);
}
- public E getInstance() {
- E instance = recycledObjects.poll();
- if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
- }
- return instance;
- }
-
- public void recycle(E object) {
- if (object != null) {
- recycledObjects.add(object);
- }
+ @Override
+ protected E unwrap(E wrapped) {
+ return wrapped;
}
@Override
- public String toString() {
- return recycledObjects.toString();
+ protected E wrap(E element) {
+ return element;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
index d0b79b1..ef9ff08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
@@ -51,7 +51,7 @@ import org.apache.hyracks.util.string.UTF8StringWriter;
public class ParserContext {
private static final int SERIALIZED_FIELDNAME_MAP_MAX_SIZE = 128;
- private final ObjectPool<IARecordBuilder, ATypeTag> objectBuilderPool;
+ private final IObjectPool<IARecordBuilder> objectBuilderPool;
private final ObjectPool<IAsterixListBuilder, ATypeTag> arrayBuilderPool;
/**
@@ -61,7 +61,7 @@ public class ParserContext {
* <p>
* Scalar value 5 is written 4 times in tempBuffer("d") then tempBuffer("c") ... tempBuffer("a")
*/
- private final ObjectPool<IMutableValueStorage, ATypeTag> tempBufferPool;
+ private final IObjectPool<IMutableValueStorage> tempBufferPool;
private final ObjectPool<BitSet, Void> nullBitmapPool;
private final Map<String, IMutableValueStorage> serializedFieldNames;
private final ISerializerDeserializer<AString> stringSerDe;
@@ -76,9 +76,9 @@ public class ParserContext {
@SuppressWarnings("unchecked")
public ParserContext(boolean allocateModfiedUTF8Writer) {
- objectBuilderPool = new ObjectPool<>(new RecordBuilderFactory());
+ objectBuilderPool = new SoftObjectPool<>(new RecordBuilderFactory());
arrayBuilderPool = new ObjectPool<>(new ListBuilderFactory(), ATypeTag.ARRAY);
- tempBufferPool = new ObjectPool<>(new AbvsBuilderFactory());
+ tempBufferPool = new SoftObjectPool<>(new AbvsBuilderFactory());
nullBitmapPool = new ObjectPool<>();
serializedFieldNames = new LRUMap<>(SERIALIZED_FIELDNAME_MAP_MAX_SIZE);
stringSerDe = SerializerDeserializerProvider.INSTANCE.getAStringSerializerDeserializer();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
similarity index 55%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
index 8945e71..91655db 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
@@ -18,8 +18,7 @@
*/
package org.apache.asterix.external.parser.jackson;
-import java.util.ArrayDeque;
-import java.util.Queue;
+import java.lang.ref.SoftReference;
import org.apache.asterix.om.util.container.IObjectFactory;
@@ -27,41 +26,26 @@ import org.apache.asterix.om.util.container.IObjectFactory;
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
- private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
-
- public ObjectPool() {
+public class SoftObjectPool<E, T> extends AbstractObjectPool<E, T, SoftReference<E>> {
+ public SoftObjectPool() {
this(null, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory) {
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory) {
this(objectFactory, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
- this.objectFactory = objectFactory;
- recycledObjects = new ArrayDeque<>();
- this.element = element;
- }
-
- public E getInstance() {
- E instance = recycledObjects.poll();
- if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
- }
- return instance;
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory, T element) {
+ super(objectFactory, element);
}
- public void recycle(E object) {
- if (object != null) {
- recycledObjects.add(object);
- }
+ @Override
+ protected E unwrap(SoftReference<E> wrapped) {
+ return wrapped == null ? null : wrapped.get();
}
@Override
- public String toString() {
- return recycledObjects.toString();
+ protected SoftReference<E> wrap(E element) {
+ return new SoftReference<>(element);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index f96ed72..38de4ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -25,6 +25,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UTFDataFormatException;
+import java.lang.ref.SoftReference;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
@@ -690,10 +691,12 @@ public class UTF8StringUtil {
if (writer == null) {
tempBytes = new byte[utflen + 5];
} else {
- if (writer.tempBytes == null || writer.tempBytes.length < utflen + 5) {
- writer.tempBytes = new byte[utflen + 5];
+ byte[] writerTempBytes = writer.tempBytesRef != null ? writer.tempBytesRef.get() : null;
+ if (writerTempBytes == null || writerTempBytes.length < utflen + 5) {
+ writerTempBytes = new byte[utflen + 5];
+ writer.tempBytesRef = new SoftReference<>(writerTempBytes);
}
- tempBytes = writer.tempBytes;
+ tempBytes = writerTempBytes;
}
return tempBytes;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
index 563f865..a0cc7d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
@@ -21,11 +21,12 @@ package org.apache.hyracks.util.string;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
+import java.lang.ref.SoftReference;
public class UTF8StringWriter implements Serializable {
private static final long serialVersionUID = 1L;
- transient byte[] tempBytes;
+ transient SoftReference<byte[]> tempBytesRef;
public final void writeUTF8(CharSequence str, DataOutput out) throws IOException {
UTF8StringUtil.writeUTF8(str, out, this);
[asterixdb] 06/38: [NO ISSUE][EXT] Provide direct access to
controller stat primitives
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 605b115c2abd9bc4570c349afde6a7c485bab32c
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Apr 1 15:08:05 2021 -0400
[NO ISSUE][EXT] Provide direct access to controller stat primitives
Change-Id: I40281b2cc682e0c32359963733909caca95f511f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10825
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/external/dataflow/AbstractFeedDataFlowController.java | 5 +++++
.../asterix/external/dataflow/FeedRecordDataFlowController.java | 5 ++++-
.../asterix/external/dataflow/FeedStreamDataFlowController.java | 1 -
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
index 94d9e6e..bd422ef 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/AbstractFeedDataFlowController.java
@@ -35,6 +35,7 @@ public abstract class AbstractFeedDataFlowController implements IDataFlowControl
protected final ArrayTupleBuilder tb;
protected final FeedLogManager feedLogManager;
protected boolean flushing;
+ protected long incomingRecordsCount = 0;
public AbstractFeedDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager, int numOfFields) {
this.feedLogManager = feedLogManager;
@@ -50,6 +51,10 @@ public abstract class AbstractFeedDataFlowController implements IDataFlowControl
flushing = false;
}
+ public long getIncomingRecordsCount() {
+ return incomingRecordsCount;
+ }
+
public abstract String getStats();
@Override
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 8cec5de..4279ebd 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -57,7 +57,6 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
protected final AtomicBoolean closed = new AtomicBoolean(false);
protected static final long INTERVAL = 1000;
protected State state = State.CREATED;
- protected long incomingRecordsCount = 0;
protected long failedRecordsCount = 0;
public FeedRecordDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager, int numOfOutputFields,
@@ -267,6 +266,10 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
return dataParser;
}
+ public long getFailedRecordsCount() {
+ return failedRecordsCount;
+ }
+
@Override
public String getStats() {
String readerStats = recordReader.getStats();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
index b42e6de..9f4a3b0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedStreamDataFlowController.java
@@ -32,7 +32,6 @@ public class FeedStreamDataFlowController extends AbstractFeedDataFlowController
private final IStreamDataParser dataParser;
private final AsterixInputStream stream;
- protected long incomingRecordsCount = 0;
public FeedStreamDataFlowController(IHyracksTaskContext ctx, FeedLogManager feedLogManager,
IStreamDataParser streamParser, AsterixInputStream inputStream) {
[asterixdb] 30/38: [NO ISSUE][STO] Close datasets of flushed
indexes after recovery
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0e7e4bdb2c514fe7469a775c2ffc3d9e78e6a317
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed May 12 16:41:02 2021 +0300
[NO ISSUE][STO] Close datasets of flushed indexes after recovery
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- After performing redo of a flush log on any index, close its dataset
to ensure any cached state that might have been changed during recovery
is cleared (e.g. the component id generator).
- Fix LSMFlushRecoveryTest total number of records to be inserted.
- Update LSMFlushRecoveryTest to check for duplicate component ids.
Change-Id: I29072f475cc7b4d7d6efde415be0329fc568443e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11423
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 20 +++++++++++++-------
.../asterix/test/dataflow/LSMFlushRecoveryTest.java | 12 ++++++++++--
.../asterix/common/api/IDatasetLifecycleManager.java | 9 +++++++++
.../common/context/DatasetLifecycleManager.java | 11 +++++++++++
.../common/transactions/IRecoveryManager.java | 11 -----------
5 files changed, 43 insertions(+), 20 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 65cb36a..0359cf1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -173,15 +173,14 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
deleteRecoveryTemporaryFiles();
//get active partitions on this node
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN, true);
}
- @Override
- public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException {
+ public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN,
+ boolean closeOnFlushRedo) throws IOException, ACIDException {
try {
Set<Long> winnerJobSet = startRecoverysAnalysisPhase(partitions, logReader, lowWaterMarkLSN);
- startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet);
+ startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet, closeOnFlushRedo);
} finally {
logReader.close();
deleteRecoveryTemporaryFiles();
@@ -277,7 +276,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
}
private synchronized void startRecoveryRedoPhase(Set<Integer> partitions, ILogReader logReader,
- long lowWaterMarkLSN, Set<Long> winnerTxnSet) throws IOException, ACIDException {
+ long lowWaterMarkLSN, Set<Long> winnerTxnSet, boolean closeOnFlushRedo) throws IOException, ACIDException {
int redoCount = 0;
long txnId = 0;
@@ -299,6 +298,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, false);
ILogRecord logRecord = null;
+ Set<Integer> flushRedoDatasets = new HashSet<>();
try {
logReader.setPosition(lowWaterMarkLSN);
logRecord = logReader.next();
@@ -409,6 +409,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
&& !index.isCurrentMutableComponentEmpty()) {
// schedule flush
redoFlush(index, logRecord);
+ flushRedoDatasets.add(datasetId);
redoCount++;
} else {
// TODO: update checkpoint file?
@@ -441,6 +442,11 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
for (long r : resourceIdList) {
datasetLifecycleManager.close(resourcesMap.get(r).getPath());
}
+ if (closeOnFlushRedo) {
+ // close datasets of indexes to ensure any cached state that might've been changed by recovery is cleared
+ // e.g. when redoing a flush, the component id generator needs to be reinitialized
+ datasetLifecycleManager.closeDatasets(flushRedoDatasets);
+ }
}
}
@@ -525,7 +531,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
if (minLSN < readableSmallestLSN) {
minLSN = readableSmallestLSN;
}
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN, false);
if (flush) {
appCtx.getDatasetLifecycleManager().flushAllDatasets();
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index 9c6e95e..2aad416 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -23,7 +23,9 @@ import java.lang.reflect.Field;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.Semaphore;
import org.apache.asterix.app.bootstrap.TestNodeController;
@@ -62,6 +64,7 @@ import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -143,8 +146,7 @@ public class LSMFlushRecoveryTest {
checkComponentIds();
// insert more records
createInsertOps();
- insertRecords(PARTITION_0, StorageTestUtils.RECORDS_PER_COMPONENT, StorageTestUtils.RECORDS_PER_COMPONENT,
- true);
+ insertRecords(PARTITION_0, StorageTestUtils.TOTAL_NUM_OF_RECORDS, StorageTestUtils.RECORDS_PER_COMPONENT, true);
dsInfo.waitForIO();
checkComponentIds();
@@ -487,8 +489,14 @@ public class LSMFlushRecoveryTest {
List<ILSMDiskComponent> secondaryDiskComponents = secondaryIndexes[partitionIndex].getDiskComponents();
Assert.assertEquals(primaryDiskComponents.size(), secondaryDiskComponents.size());
+ Set<ILSMComponentId> uniqueIds = new HashSet<>();
for (int i = 0; i < primaryDiskComponents.size(); i++) {
Assert.assertEquals(primaryDiskComponents.get(i).getId(), secondaryDiskComponents.get(i).getId());
+ ILSMComponentId id = primaryDiskComponents.get(i).getId();
+ boolean added = uniqueIds.add(id);
+ if (!added) {
+ throw new IllegalStateException("found duplicate component ids: " + id);
+ }
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
index 7b737a0..b03af55 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common.api;
import java.util.List;
+import java.util.Set;
import java.util.function.Predicate;
import org.apache.asterix.common.context.DatasetInfo;
@@ -124,6 +125,14 @@ public interface IDatasetLifecycleManager extends IResourceLifecycleManager<IInd
List<IVirtualBufferCache> getVirtualBufferCaches(int datasetId, int ioDeviceNum);
/**
+ * Attempts to close the datasets in {@code datasetsToClose}
+ *
+ * @param datasetsToClose
+ * @throws HyracksDataException
+ */
+ void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException;
+
+ /**
* Flushes then closes all open datasets
*/
void closeAllDatasets() throws HyracksDataException;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index b26220d..b2f4034 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -25,6 +25,7 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
@@ -452,6 +453,16 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC
}
@Override
+ public synchronized void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException {
+ ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
+ for (DatasetResource dsr : openDatasets) {
+ if (dsr.isOpen() && datasetsToClose.contains(dsr.getDatasetID())) {
+ closeDataset(dsr);
+ }
+ }
+ }
+
+ @Override
public synchronized void closeAllDatasets() throws HyracksDataException {
ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
for (DatasetResource dsr : openDatasets) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
index bfe7963..8a5f34e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
@@ -85,17 +85,6 @@ public interface IRecoveryManager {
long getLocalMinFirstLSN() throws HyracksDataException;
/**
- * Replay the logs that belong to the passed {@code partitions} starting from the {@code lowWaterMarkLSN}
- *
- * @param partitions
- * @param lowWaterMarkLSN
- * @throws IOException
- * @throws ACIDException
- */
- void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException;
-
- /**
* Creates a temporary file to be used during recovery
*
* @param txnId
[asterixdb] 16/38: [NO ISSUE] Update third-party dependencies
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3856cb39f60d86f492330bd42e899c2cd13c5813
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat May 1 11:36:15 2021 -0400
[NO ISSUE] Update third-party dependencies
- com.fasterxml.jackson.core:jackson-*:2.12.1 -> 2.12.3
- commons-codec:commons-codec:1.14 -> 1.15
- commons-io:commons-io:2.6 -> 2.8.0
- io.netty:netty-all:4.1.59.Final -> 4.1.63.Final
- org.apache.commons:commons-lang3:3.9 -> 3.12.0
- org.apache.commons:commons-text:1.8 -> 1.9
- org.apache.httpcomponents:httpcore:4.4.13 -> 4.4.14
- org.apache.logging.log4j:log4j-*:2.13.3 -> 2.14.1
Change-Id: I9aa3ea779ccc84ab2fafbf33345d391f34f0696d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11263
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
asterixdb/asterix-server/pom.xml | 4 ++--
asterixdb/pom.xml | 4 ++--
.../main/appended-resources/supplemental-models.xml | 8 ++++----
...tent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
hyracks-fullstack/pom.xml | 18 +++++++++---------
5 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 877293c..ee58e2d 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -169,8 +169,8 @@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
- <gav>io.netty:netty-all:4.1.59.Final</gav>
- <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.59.Final/NOTICE.txt</noticeUrl>
+ <gav>io.netty:netty-all:4.1.63.Final</gav>
+ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.63.Final/NOTICE.txt</noticeUrl>
</override>
</overrides>
<licenses>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 81e53c5..ed486fd 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -77,7 +77,7 @@
<hyracks.version>0.3.4.2-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -1281,7 +1281,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.14</version>
+ <version>1.15</version>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index 3f176a7..dd56718 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -160,9 +160,9 @@
<artifactId>netty-all</artifactId>
<properties>
<!-- netty is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <license.ignoreMissingEmbeddedLicense>4.1.59.Final</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>4.1.59.Final</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>4.1.59.Final</license.ignoreNoticeOverride>
+ <license.ignoreMissingEmbeddedLicense>4.1.63.Final</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>4.1.63.Final</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreNoticeOverride>4.1.63.Final</license.ignoreNoticeOverride>
</properties>
</project>
</supplement>
@@ -229,7 +229,7 @@
<artifactId>jackson-annotations</artifactId>
<properties>
<!-- jackson-annotations does not provide an embedded NOTICE file -->
- <license.ignoreMissingEmbeddedNotice>2.8.4,2.9.7,2.9.10,2.10.0,2.10.3,2.12.1</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreMissingEmbeddedNotice>2.12.3</license.ignoreMissingEmbeddedNotice>
</properties>
</project>
</supplement>
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt
similarity index 100%
rename from asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt
rename to asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 2f34980..0ac307d 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -69,7 +69,7 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
<implementation.title>Apache Hyracks and Algebricks - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
@@ -81,7 +81,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.1.59.Final</version>
+ <version>4.1.63.Final</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -142,27 +142,27 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.6</version>
+ <version>2.8.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
- <version>1.8</version>
+ <version>1.9</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.12.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -194,7 +194,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.9</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -204,7 +204,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.4.13</version>
+ <version>4.4.14</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
[asterixdb] 18/38: Merge branch 'gerrit/stabilization-5949a1cb71'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6abbeeaff95cd6a7d1ce27df7b94fdff9e9c5aa1
Merge: 3ea8ef1 2fe0e14
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun May 2 09:27:39 2021 -0400
Merge branch 'gerrit/stabilization-5949a1cb71'
Change-Id: Id8c5f6de8ce3738a89994e87c3fc6d0ca7150d8e
asterixdb/asterix-server/pom.xml | 4 ++--
asterixdb/pom.xml | 4 ++--
.../main/appended-resources/supplemental-models.xml | 8 ++++----
...tent.com_netty_netty_netty-4.1.63.Final_NOTICE.txt} | 0
hyracks-fullstack/pom.xml | 18 +++++++++---------
5 files changed, 17 insertions(+), 17 deletions(-)
diff --cc asterixdb/asterix-server/pom.xml
index ba0abdd,5a279d0..266866e
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@@ -169,38 -169,9 +169,38 @@@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
- <gav>io.netty:netty-all:4.1.59.Final</gav>
- <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.59.Final/NOTICE.txt</noticeUrl>
+ <gav>io.netty:netty-all:4.1.63.Final</gav>
+ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.63.Final/NOTICE.txt</noticeUrl>
</override>
+ <override>
+ <gav>org.reactivestreams:reactive-streams:1.0.2</gav>
+ <noticeUrl>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/COPYING.txt</noticeUrl>
+ <url>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/LICENSE.txt</url>
+ </override>
+ <override>
+ <gavs>
+ <gav>software.amazon.awssdk:sdk-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:auth:2.10.83</gav>
+ <gav>software.amazon.awssdk:arns:2.10.83</gav>
+ <gav>software.amazon.awssdk:annotations:2.10.83</gav>
+ <gav>software.amazon.awssdk:utils:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-query-protocol:2.10.83</gav>
+ <gav>software.amazon.awssdk:profiles:2.10.83</gav>
+ <gav>software.amazon.awssdk:protocol-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:s3:2.10.83</gav>
+ <gav>software.amazon.awssdk:netty-nio-client:2.10.83</gav>
+ <gav>software.amazon.awssdk:apache-client:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-xml-protocol:2.10.83</gav>
+ <gav>software.amazon.awssdk:regions:2.10.83</gav>
+ <gav>software.amazon.awssdk:http-client-spi:2.10.83</gav>
+ </gavs>
+ <noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl>
+ </override>
+ <override>
+ <gav>software.amazon.eventstream:eventstream:1.0.1</gav>
+ <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-eventstream-java/7be2dd80e12f8835674c8ffb0f4a2efb64c7b585/NOTICE</noticeUrl>
+ </override>
</overrides>
<licenses>
<license>
diff --cc asterixdb/pom.xml
index 5b7e6c9,36b4581..43e0dfd
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -81,8 -81,7 +81,8 @@@
<hyracks.version>0.3.5-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.13.3</log4j.version>
+ <log4j.version>2.14.1</log4j.version>
+ <awsjavasdk.version>2.10.83</awsjavasdk.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>
[asterixdb] 28/38: [NO ISSUE] Fixed storage unit util false
positives + added tests
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d5de9eb65fa935076d0bb82094d3fdf5357ea704
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Tue May 11 17:09:05 2021 +0300
[NO ISSUE] Fixed storage unit util false positives + added tests
Change-Id: Ic263c4fea0de15803811c88e4d6b01df21e083c8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11303
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../java/org/apache/hyracks/util/StorageUtil.java | 53 ++++++---------
.../org/apache/hyracks/util/StorageUnitTest.java | 79 ++++++++++++++++++++++
2 files changed, 99 insertions(+), 33 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
index e4969f0..0456dc7 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
@@ -20,10 +20,13 @@ package org.apache.hyracks.util;
import java.util.HashMap;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class StorageUtil {
public static final int BASE = 1024;
+ private static final Pattern PATTERN = Pattern.compile("^(-?[.0-9]+)([A-Z]{0,2})$");
public enum StorageUnit {
BYTE("B", "b", 1),
@@ -96,43 +99,27 @@ public class StorageUtil {
* @throws IllegalArgumentException
*/
public static double getSizeInBytes(String s) {
- String sSpaceRemoved = s.replaceAll(" ", "");
- String sUpper = sSpaceRemoved.toUpperCase();
-
- // Default type
- StorageUtil.StorageUnit unitType;
-
- // If the length is 1, it should only contain a digit number.
- if (sUpper.length() == 1) {
- if (Character.isDigit(sUpper.charAt(0))) {
- unitType = StorageUnit.BYTE;
- } else {
- throw invalidFormatException(s);
- }
- } else if (sUpper.length() > 1) {
- String checkStr = sUpper.substring(sUpper.length() - 2);
- unitType = StorageUnit.lookupBySuffix(checkStr);
-
- if (unitType == null) {
- // The last suffix should be at least "B" or a digit to be qualified as byte unit string.
- char lastChar = sUpper.charAt(sUpper.length() - 1);
- if (sUpper.substring(sUpper.length() - 1).equals(StorageUnit.BYTE.toString())
- || Character.isDigit(lastChar)) {
- unitType = StorageUnit.BYTE;
- } else {
- throw invalidFormatException(s);
- }
- }
- } else {
- // String length is zero. We can't parse this string.
+ String valueAndUnit = s.replace(" ", "").toUpperCase();
+ Matcher matcher = PATTERN.matcher(valueAndUnit);
+ if (!matcher.find()) {
throw invalidFormatException(s);
}
- // Strip all unit suffixes such as KB, MB ...
- String sFinalVal = sUpper.replaceAll("[^-\\.0123456789]", "");
+ String value = matcher.group(1);
+ String unit = matcher.group(2);
+
+ // Default to bytes or find provided unit
+ StorageUnit unitType = !unit.isEmpty() ? StorageUnit.lookupBySuffix(unit) : StorageUnit.BYTE;
+ if (unitType == null) {
+ throw invalidFormatException(s);
+ }
- // Return the bytes.
- return unitType.toBytes(Double.parseDouble(sFinalVal));
+ try {
+ // Return the bytes.
+ return unitType.toBytes(Double.parseDouble(value));
+ } catch (NumberFormatException ex) {
+ throw invalidFormatException(s);
+ }
}
private static IllegalArgumentException invalidFormatException(String s) {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java
new file mode 100644
index 0000000..445d15f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StorageUnitTest {
+
+ @Test
+ public void test() {
+ // Valid cases
+ double result1NoUnit = StorageUtil.getSizeInBytes("1"); // Defaults to bytes
+ Assert.assertEquals(1.0, result1NoUnit, 0);
+
+ double result1B = StorageUtil.getSizeInBytes("1B");
+ Assert.assertEquals(1.0, result1B, 0);
+
+ double result1BWithSpaces = StorageUtil.getSizeInBytes("1 B ");
+ Assert.assertEquals(1.0, result1BWithSpaces, 0);
+
+ double result1Kb = StorageUtil.getSizeInBytes("1KB");
+ Assert.assertEquals(1024.0, result1Kb, 0);
+
+ double result1KbWithSpaces = StorageUtil.getSizeInBytes(" 1 K B ");
+ Assert.assertEquals(1024.0, result1KbWithSpaces, 0);
+
+ double resultPoint5KB = StorageUtil.getSizeInBytes(".5KB");
+ Assert.assertEquals(512.0, resultPoint5KB, 0);
+
+ double resultPoint5SmallKB = StorageUtil.getSizeInBytes(".5kB");
+ Assert.assertEquals(512.0, resultPoint5SmallKB, 0);
+
+ double result1Mb = StorageUtil.getSizeInBytes("1MB");
+ Assert.assertEquals(1024.0 * 1024.0, result1Mb, 0);
+
+ double result1Point0Mb = StorageUtil.getSizeInBytes("1.0MB");
+ Assert.assertEquals(1024.0 * 1024.0, result1Point0Mb, 0);
+
+ double result01Point0Mb = StorageUtil.getSizeInBytes("01.0MB");
+ Assert.assertEquals(1024.0 * 1024.0, result01Point0Mb, 0);
+
+ // Invalid cases
+ invalidCase("");
+ invalidCase("99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999");
+ invalidCase("32MB123");
+ invalidCase("1.1.1");
+ invalidCase("12KBMB");
+ invalidCase("MB");
+ invalidCase("1AB");
+ invalidCase("MB1MB");
+ invalidCase("123MBB");
+ }
+
+ private void invalidCase(String value) {
+ try {
+ StorageUtil.getSizeInBytes(value);
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.toString()
+ .contains("IllegalArgumentException: The given string: " + value + " is not a byte unit string"));
+ }
+ }
+}
[asterixdb] 15/38: [NO ISSUE][OTH] Add util method to get request
http scheme
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 82f19315dde3cf25191314ea558ba0e7d3c474d2
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Thu Apr 29 17:44:38 2021 +0300
[NO ISSUE][OTH] Add util method to get request http scheme
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add util method to get request http scheme.
- Allow exceptions to be thrown from servletNotFound.
Change-Id: I85e88223a0ec3291be4423cf5dc146d7d7d09903
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11224
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../java/org/apache/hyracks/http/server/HttpServerHandler.java | 8 ++------
.../main/java/org/apache/hyracks/http/server/utils/HttpUtil.java | 6 ++++++
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
index 77d3493..a7ace7a 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
@@ -18,8 +18,6 @@
*/
package org.apache.hyracks.http.server;
-import static org.apache.hyracks.http.server.utils.HttpUtil.X_FORWARDED_PROTO;
-
import java.io.IOException;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
@@ -143,9 +141,7 @@ public class HttpServerHandler<T extends HttpServer> extends SimpleChannelInboun
private void submit(ChannelHandlerContext ctx, IServlet servlet, FullHttpRequest request) throws IOException {
IServletRequest servletRequest;
try {
- HttpScheme scheme =
- server.getScheme() == HttpScheme.HTTPS || "https".equals(request.headers().get(X_FORWARDED_PROTO))
- ? HttpScheme.HTTPS : HttpScheme.HTTP;
+ HttpScheme scheme = HttpUtil.getScheme(server, request);
servletRequest = createServletRequest(ctx, request, scheme);
} catch (IllegalArgumentException e) {
LOGGER.log(Level.WARN, "Failure Decoding Request", e);
@@ -166,7 +162,7 @@ public class HttpServerHandler<T extends HttpServer> extends SimpleChannelInboun
}
}
- protected void handleServletNotFound(ChannelHandlerContext ctx, FullHttpRequest request) {
+ protected void handleServletNotFound(ChannelHandlerContext ctx, FullHttpRequest request) throws IOException {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("No servlet for " + request.uri());
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 4c0942b..c61e0d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -43,6 +43,7 @@ import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.BaseRequest;
import org.apache.hyracks.http.server.FormUrlEncodedRequest;
+import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.util.ThrowingConsumer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -230,6 +231,11 @@ public class HttpUtil {
}
}
+ public static HttpScheme getScheme(HttpServer server, FullHttpRequest request) {
+ return server.getScheme() == HttpScheme.HTTPS || "https".equals(request.headers().get(X_FORWARDED_PROTO))
+ ? HttpScheme.HTTPS : HttpScheme.HTTP;
+ }
+
public static class ContentType {
public static final String ADM = "adm";
public static final String JSON = "json";
[asterixdb] 26/38: [ASTERIXDB-2896] Increase UDF argument buffer
size
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a0d1fb7acb588f4eab940501deee738789e9af50
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Sun May 2 11:23:41 2021 -0700
[ASTERIXDB-2896] Increase UDF argument buffer size
-user model changes: no
-storage format changes: no
-interface changes: yes
Details:
Bump buffer sizes in Python IPC to 1MB for individual
arguments, and in the case of batching, match the
buffer size of the Hyracks IPC layer for deserialization.
Change-Id: If847ac3b09406d1e9e6a976a7e0193b6e81bcc8b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11243
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Ian Maxon <im...@uci.edu>
---
.../ExternalScalarPythonFunctionEvaluator.java | 9 ++++++---
.../operators/ExternalAssignBatchRuntimeFactory.java | 19 ++++++++++++++++---
.../asterix/external/util/ExternalDataUtils.java | 19 +++++++++++++++++++
3 files changed, 41 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index eb87399..7c860a2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -29,6 +29,7 @@ import java.nio.ByteBuffer;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -77,9 +78,11 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua
for (int i = 0; i < argValues.length; i++) {
argValues[i] = VoidPointable.FACTORY.createPointable();
}
- //TODO: these should be dynamic
- this.argHolder = ByteBuffer.wrap(new byte[Short.MAX_VALUE * 2]);
- this.outputWrapper = ByteBuffer.wrap(new byte[Short.MAX_VALUE * 2]);
+ //TODO: these should be dynamic. this static size picking is a temporary bodge until this works like
+ // v-size frames do or these construction buffers are removed entirely
+ int maxArgSz = ExternalDataUtils.getArgBufferSize();
+ this.argHolder = ByteBuffer.wrap(new byte[maxArgSz]);
+ this.outputWrapper = ByteBuffer.wrap(new byte[maxArgSz]);
this.evaluatorContext = ctx;
this.sourceLocation = sourceLoc;
this.unpackerInput = new ArrayBufferInput(new byte[0]);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
index 39e480a..593bac6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
@@ -36,6 +36,7 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.library.PythonLibraryEvaluator;
import org.apache.asterix.external.library.PythonLibraryEvaluatorFactory;
import org.apache.asterix.external.library.msgpack.MessageUnpackerToADM;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -61,6 +62,8 @@ public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOne
private final IExternalFunctionDescriptor[] fnDescs;
private final int[][] fnArgColumns;
+ private int rpcBufferSize;
+
public ExternalAssignBatchRuntimeFactory(int[] outColumns, IExternalFunctionDescriptor[] fnDescs,
int[][] fnArgColumns, int[] projectionList) {
super(projectionList);
@@ -73,6 +76,9 @@ public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOne
public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(IHyracksTaskContext ctx) {
final int[] projectionToOutColumns = new int[projectionList.length];
+ //this is a temporary bodge. these buffers need to work like vsize frames, or be absent entirely
+ int maxArgSz = ExternalDataUtils.getArgBufferSize();
+ rpcBufferSize = ExternalDataUtils.roundUpToNearestFrameSize(maxArgSz, ctx.getInitialFrameSize());
for (int j = 0; j < projectionList.length; j++) {
projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
}
@@ -110,14 +116,14 @@ public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOne
}
argHolders = new ArrayList<>(fnArgColumns.length);
for (int i = 0; i < fnArgColumns.length; i++) {
- argHolders.add(ctx.allocateFrame());
+ argHolders.add(ctx.allocateFrame(rpcBufferSize));
}
outputWrapper = ctx.allocateFrame();
nullCalls = new ATypeTag[argHolders.size()][0];
numCalls = new int[fnArgColumns.length];
batchResults = new ArrayList<>(argHolders.size());
for (int i = 0; i < argHolders.size(); i++) {
- batchResults.add(new Pair<>(ctx.allocateFrame(), new Counter(-1)));
+ batchResults.add(new Pair<>(ctx.allocateFrame(rpcBufferSize), new Counter(-1)));
}
unpackerInput = new ArrayBufferInput(new byte[0]);
unpacker = MessagePack.newDefaultUnpacker(unpackerInput);
@@ -230,7 +236,8 @@ public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOne
if (columnResult != null) {
Pair<ByteBuffer, Counter> resultholder = batchResults.get(argHolderIdx);
if (resultholder.getFirst().capacity() < columnResult.capacity()) {
- resultholder.setFirst(ctx.allocateFrame(columnResult.capacity()));
+ resultholder.setFirst(ctx.allocateFrame(ExternalDataUtils.roundUpToNearestFrameSize(
+ columnResult.capacity(), ctx.getInitialFrameSize())));
}
ByteBuffer resultBuf = resultholder.getFirst();
resultBuf.clear();
@@ -262,6 +269,12 @@ public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOne
outputWrapper.clear();
outputWrapper.position(0);
Pair<ByteBuffer, Counter> result = batchResults.get(k);
+ if (result.getFirst() != null) {
+ if (result.getFirst().capacity() > outputWrapper.capacity()) {
+ outputWrapper = ctx.allocateFrame(ExternalDataUtils.roundUpToNearestFrameSize(
+ outputWrapper.capacity(), ctx.getInitialFrameSize()));
+ }
+ }
int start = outputWrapper.arrayOffset();
ATypeTag functionCalled = nullCalls[k][i];
if (functionCalled == ATypeTag.TYPE) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index f5c62a5..8e94263 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -80,6 +80,7 @@ import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.LongParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import org.apache.hyracks.util.StorageUtil;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
@@ -106,6 +107,8 @@ import software.amazon.awssdk.services.s3.model.S3Response;
public class ExternalDataUtils {
private static final Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = new EnumMap<>(ATypeTag.class);
+ private static final int DEFAULT_MAX_ARGUMENT_SZ = 1024 * 1024;
+ private static final int HEADER_FUDGE = 64;
static {
valueParserFactoryMap.put(ATypeTag.INTEGER, IntegerParserFactory.INSTANCE);
@@ -997,4 +1000,20 @@ public class ExternalDataUtils {
}
}
}
+
+ public static int roundUpToNearestFrameSize(int size, int framesize) {
+ return ((size / framesize) + 1) * framesize;
+ }
+
+ public static int getArgBufferSize() {
+ int maxArgSz = DEFAULT_MAX_ARGUMENT_SZ + HEADER_FUDGE;
+ String userArgSz = System.getProperty("udf.buf.size");
+ if (userArgSz != null) {
+ long parsedSize = StorageUtil.getByteValue(userArgSz) + HEADER_FUDGE;
+ if (parsedSize < Integer.MAX_VALUE && parsedSize > 0) {
+ maxArgSz = (int) parsedSize;
+ }
+ }
+ return maxArgSz;
+ }
}
[asterixdb] 31/38: [ASTERIXDB-2884] Compiler error with nested UDF
calls
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 86da3f820f9e59fe5382dc42c54b0828cc9aecb8
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Wed May 12 22:25:50 2021 -0700
[ASTERIXDB-2884] Compiler error with nested UDF calls
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix invalid plan produced by ExtractBatchableExternalFunctionCallsRule
- IntroduceDynamicTypeCastForExternalFunctionRule should not introduce
redundant cast if function argument type is ANY
- Add testcase
Change-Id: I472aa8f828cd548d14d7bda9893984cc4bf25737
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11445
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../rules/ExtractBatchableExternalFunctionCallsRule.java | 12 +++++-------
.../IntroduceDynamicTypeCastForExternalFunctionRule.java | 4 ++++
.../{mysentiment.0.ddl.sqlpp => toplevel_fn.0.ddl.sqlpp} | 0
.../{mysentiment.1.lib.sqlpp => toplevel_fn.1.lib.sqlpp} | 0
.../{mysentiment.2.ddl.sqlpp => toplevel_fn.2.ddl.sqlpp} | 0
.../{mysentiment.3.query.sqlpp => toplevel_fn.3.query.sqlpp} | 0
.../{mysentiment.3.query.sqlpp => toplevel_fn.4.query.sqlpp} | 7 ++++++-
.../{mysentiment.4.ddl.sqlpp => toplevel_fn.5.ddl.sqlpp} | 0
.../toplevel_fn/{toplevel_fn.1.adm => toplevel_fn.3.adm} | 0
.../results/external-library/toplevel_fn/toplevel_fn.4.adm | 1 +
10 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
index 1e5d805..54a1bb7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
@@ -190,13 +190,11 @@ public class ExtractBatchableExternalFunctionCallsRule implements IAlgebraicRewr
if (!assignVars.isEmpty()) {
usedVarList.clear();
expr.getUsedVariables(usedVarList);
- for (int i = 0, ln = assignVars.size(); i < ln; i++) {
- List<LogicalVariable> candidateVarList = assignVars.get(i);
- if (OperatorPropertiesUtil.disjoint(candidateVarList, usedVarList)) {
- assignVarList = candidateVarList;
- assignExprList = assignExprs.get(i);
- break;
- }
+ int candidateVarListIdx = assignVars.size() - 1;
+ List<LogicalVariable> candidateVarList = assignVars.get(candidateVarListIdx);
+ if (OperatorPropertiesUtil.disjoint(candidateVarList, usedVarList)) {
+ assignVarList = candidateVarList;
+ assignExprList = assignExprs.get(candidateVarListIdx);
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
index 2c5a07d..22d87ac 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
@@ -22,6 +22,7 @@ package org.apache.asterix.optimizer.rules;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
@@ -82,6 +83,9 @@ public class IntroduceDynamicTypeCastForExternalFunctionRule implements IAlgebra
if (reqArgType.getTypeTag() == ATypeTag.OBJECT) {
castFlag = !IntroduceDynamicTypeCastRule.compatible((ARecordType) reqArgType, inputType,
argExpr.getValue().getSourceLocation());
+ } else if (reqArgType.getTypeTag() == ATypeTag.ANY) {
+ IAType inputPrimeType = TypeComputeUtils.getActualType(inputType);
+ castFlag = inputPrimeType.getTypeTag().isDerivedType();
} else {
castFlag = !reqArgType.equals(inputType);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.0.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.0.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.0.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.1.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.1.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.1.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.2.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.2.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.2.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.3.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.3.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp
similarity index 93%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp
index 755d980..7ebfc7a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.4.query.sqlpp
@@ -16,7 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
+
+/*
+ * Testcase for ASTERIXDB-2884
+ */
+
use externallibtest;
-sqrt(4);
+sqrt(sqrt(null));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.5.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/mysentiment.4.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/toplevel_fn/toplevel_fn.5.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.3.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.1.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/toplevel_fn/toplevel_fn.4.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file