You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/04/06 18:21:05 UTC
Change in asterixdb[cheshire-cat]: merge mad-hatter into cheshire-cat
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926 )
Change subject: merge mad-hatter into cheshire-cat
......................................................................
merge mad-hatter into cheshire-cat
Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
D asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
26 files changed, 16 insertions(+), 433 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/26/10926/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index a091905..9683b9d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -130,12 +130,12 @@
public static final String PREFIX_INTERNAL_PARAMETERS = "_internal";
// A white list of supported configurable parameters.
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES = ImmutableSet.of(
CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY,
CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
@@ -147,20 +147,6 @@
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK);
-=======
- private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
- ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
- CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
- CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
- CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
- FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
- StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
- FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
- SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
- AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION,
- DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
deleted file mode 100644
index f8e48d7..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ /dev/null
@@ -1,263 +0,0 @@
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
- <test-group name="external-dataset">
- <test-case FilePath="external-dataset/aws/s3/">
- <compilation-unit name="custom-buffer-size">
- <output-dir compare="Text">custom-buffer-size</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/json">
- <output-dir compare="Text">aws/s3/json/json</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/gz">
- <output-dir compare="Text">aws/s3/json/gz</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/mixed">
- <output-dir compare="Text">aws/s3/json/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/csv">
- <output-dir compare="Text">aws/s3/csv/csv</output-dir>
- </compilation-unit>
- </test-case><test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/gz">
- <output-dir compare="Text">aws/s3/csv/gz</output-dir>
- </compilation-unit>
- </test-case><test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/mixed">
- <output-dir compare="Text">aws/s3/csv/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/tsv">
- <output-dir compare="Text">aws/s3/tsv/tsv</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/gz">
- <output-dir compare="Text">aws/s3/tsv/gz</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/mixed">
- <output-dir compare="Text">aws/s3/tsv/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/empty-string-definition">
- <output-dir compare="Text">aws/s3/empty-string-definition</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/over-1000-objects">
- <output-dir compare="Text">aws/s3/over-1000-objects</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/malformed-json">
- <output-dir compare="Text">aws/s3/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/definition-does-not-exist">
- <output-dir compare="Text">aws/s3/definition-does-not-exist</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/invalid-endpoint">
- <output-dir compare="Text">aws/s3/invalid-endpoint</output-dir>
- <expected-error>External source error. Invalid service endpoint ^invalid-endpoint^</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/bucket-does-not-exist">
- <output-dir compare="Text">aws/s3/bucket-does-not-exist</output-dir>
- <expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null)</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/definition-points-to-nothing">
- <output-dir compare="Text">aws/s3/no-files-returned/definition-points-to-nothing</output-dir>
- <source-location>false</source-location>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/exclude-all-files">
- <output-dir compare="Text">aws/s3/no-files-returned/exclude-all-files</output-dir>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/include-no-files">
- <output-dir compare="Text">aws/s3/no-files-returned/include-no-files</output-dir>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/non-s3-region">
- <output-dir compare="Text">aws/s3/non-s3-region</output-dir>
- </compilation-unit>
- </test-case>
- </test-group>
- <test-group name="include-exclude">
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-1">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-1</output-dir>
- <expected-error>Invalid format for property "exclude1"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-2">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-2</output-dir>
- <expected-error>Invalid format for property "exclude#"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-3">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-3</output-dir>
- <expected-error>Invalid format for property "exclude#hello"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/both">
- <output-dir compare="Text">aws/s3/include-exclude/both</output-dir>
- <expected-error>The parameters "include" and "exclude" cannot be provided at the same time</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-all">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-all</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-1">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-1</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-2">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-2</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-3">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-3</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-4">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-4</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-5">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-5</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-6">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-6</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-all">
- <output-dir compare="Text">aws/s3/include-exclude/include-all</output-dir>
- <expected-error>Malformed input stream</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-1">
- <output-dir compare="Text">aws/s3/include-exclude/include-1</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-2">
- <output-dir compare="Text">aws/s3/include-exclude/include-2</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-3">
- <output-dir compare="Text">aws/s3/include-exclude/include-3</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-4">
- <output-dir compare="Text">aws/s3/include-exclude/include-4</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-5">
- <output-dir compare="Text">aws/s3/include-exclude/include-5</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-6">
- <output-dir compare="Text">aws/s3/include-exclude/include-6</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-7">
- <output-dir compare="Text">aws/s3/include-exclude/include-7</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-8">
- <output-dir compare="Text">aws/s3/include-exclude/include-8</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-9">
- <output-dir compare="Text">aws/s3/include-exclude/include-9</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-10">
- <output-dir compare="Text">aws/s3/include-exclude/include-10</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-11">
- <output-dir compare="Text">aws/s3/include-exclude/include-11</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-12">
- <output-dir compare="Text">aws/s3/include-exclude/include-12</output-dir>
- </compilation-unit>
- </test-case>
- </test-group>
-</test-suite>
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 765ccd0..60ecf7f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -130,14 +130,11 @@
public Object defaultValue() {
return defaultValue;
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
@Override
public boolean hidden() {
- return this == COMPILER_STRINGOFFSET || this == COMPILER_EXTERNALSCANMEMORY;
+ return this == COMPILER_EXTERNALSCANMEMORY;
}
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini();
@@ -156,7 +153,6 @@
public static final String COMPILER_SORT_SAMPLES_KEY = Option.COMPILER_SORT_SAMPLES.ini();
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final String COMPILER_INDEXONLY_KEY = Option.COMPILER_INDEXONLY.ini();
public static final String COMPILER_INTERNAL_SANITYCHECK_KEY = Option.COMPILER_INTERNAL_SANITYCHECK.ini();
@@ -168,9 +164,8 @@
public static final String COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY = Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN.ini();
public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
-=======
+
public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
@@ -214,7 +209,6 @@
return accessor.getInt(Option.COMPILER_SORT_SAMPLES);
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public boolean isIndexOnly() {
return accessor.getBoolean(Option.COMPILER_INDEXONLY);
}
@@ -237,9 +231,9 @@
public boolean getMinMemoryAllocation() {
return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
-=======
+ }
+
public int getExternalScanMemorySize() {
return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index cb0e799..c4dbd85 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -58,7 +58,6 @@
compilerProperties.getWindowMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_WINDOW, sourceLoc);
int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc);
int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
boolean fullParallelSort = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
compilerProperties.getSortParallel());
boolean indexOnly = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_INDEXONLY_KEY,
@@ -73,13 +72,9 @@
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
boolean minMemoryAllocation = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
-=======
- boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig);
int externalScanBufferSize = getExternalScanBufferSize(
(String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
compilerProperties.getExternalScanMemorySize(), sourceLoc);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
@@ -89,17 +84,13 @@
physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
physOptConf.setSortParallel(fullParallelSort);
physOptConf.setSortSamples(sortNumSamples);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
physOptConf.setIndexOnly(indexOnly);
physOptConf.setSanityCheckEnabled(sanityCheck);
physOptConf.setExternalFieldPushdown(externalFieldPushdown);
physOptConf.setSubplanMerge(subplanMerge);
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
-=======
physOptConf.setExternalScanBufferSize(externalScanBufferSize);
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
return physOptConf;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index acce233..54a2987 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -28,73 +26,37 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-=======
-import org.apache.asterix.external.input.stream.AbstractMultipleInputStream;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public class AwsS3InputStream extends AbstractExternalInputStream {
-=======
-public class AwsS3InputStream extends AbstractMultipleInputStream {
private static final Logger LOGGER = LogManager.getLogger();
// Configuration
- private final String bucket;
private final int bufferSize;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
private final S3Client s3Client;
private final String bucket;
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
super(configuration, filePaths);
-=======
- this.filePaths = filePaths;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
this.s3Client = buildAwsS3Client(configuration);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
this.bucket = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-=======
this.bufferSize = ExternalDataUtils.getOrDefaultBufferSize(configuration);
- this.bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
@Override
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
protected boolean getInputStream() throws IOException {
-=======
- protected boolean advance() throws IOException {
- // No files to read for this partition
- if (filePaths == null || filePaths.isEmpty()) {
- return false;
- }
-
- // Finished reading all the files
- if (nextFileIndex >= filePaths.size()) {
- return false;
- }
-
- // Close the current stream before going to the next one
- if (in != null) {
- CleanupUtils.close(in, null);
- }
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(fileName).build();
@@ -112,21 +74,10 @@
}
// Use gzip stream if needed
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
String lowerCaseFileName = fileName.toLowerCase();
if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
- in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
-=======
- if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
in = new GZIPInputStream(in, bufferSize);
}
-
- // Current file ready, point to the next file
- nextFileIndex++;
- if (notificationHandler != null) {
- notificationHandler.notifyNewSource();
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
- }
return true;
}
@@ -137,41 +88,4 @@
throw HyracksDataException.create(ex);
}
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
-
- @Override
- public boolean stop() {
- return false;
- }
-
- @Override
- public boolean handleException(Throwable th) {
- return false;
- }
-
- @Override
- public void close() throws IOException {
- if (in != null) {
- CleanupUtils.close(in, null);
- }
- if (s3Client != null) {
- CleanupUtils.close(s3Client, null);
- }
- }
-
- @Override
- public String getStreamName() {
- return getStreamNameAt(nextFileIndex - 1);
- }
-
- @Override
- public String getPreviousStreamName() {
- return getStreamNameAt(nextFileIndex - 2);
- }
-
- private String getStreamNameAt(int fileIndex) {
- return fileIndex < 0 || filePaths == null || filePaths.isEmpty() ? "" : filePaths.get(fileIndex);
- }
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index 8c4488d..0098778 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -27,26 +27,18 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AsterixInputStreamReader extends Reader {
private final AsterixInputStream in;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private final CharsetDecoder decoder;
- private final byte[] bytes;
- protected final ByteBuffer byteBuffer;
- protected final CharBuffer charBuffer;
+ private byte[] bytes;
+ protected ByteBuffer byteBuffer;
+ protected CharBuffer charBuffer;
protected boolean done = false;
protected boolean remaining = false;
-=======
- private final byte[] bytes;
- private final ByteBuffer byteBuffer;
- private final CharBuffer charBuffer;
- private final CharsetDecoder decoder;
- private boolean done = false;
- private boolean remaining = false;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
public AsterixInputStreamReader(AsterixInputStream in, int bufferSize) {
this.in = in;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index a415ddb..088cd58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -119,11 +119,8 @@
// a string representing the NULL value
public static final String KEY_NULL_STR = "null";
public static final String KEY_REDACT_WARNINGS = "redact-warnings";
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final String KEY_REQUESTED_FIELDS = "requested-fields";
-=======
public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
/**
* Keys for adapter name
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index c89b669..dffde9c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -18,12 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
import java.util.HashMap;
-=======
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
import java.util.List;
import java.util.Map;
@@ -115,21 +111,14 @@
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
- Map<String, String> properties = addProjectionInfo(projectionInfo, edd.getProperties());
- ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
- edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
- tupleFilterFactory, outputLimit);
-=======
PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
Map<String, String> properties = edd.getProperties();
properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
+ return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
+ tupleFilterFactory, outputLimit);
case INTERNAL:
DataSourceId id = getId();
DataverseName dataverseName = id.getDataverseName();
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index ba6f96b..c6a79a5 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -102,19 +102,11 @@
serializerDeserializerProvider, hashFunctionFactoryProvider, hashFunctionFamilyProvider,
comparatorFactoryProvider, typeTraitProvider, binaryBooleanInspectorFactory,
binaryIntegerInspectorFactory, printerProvider, missingWriterFactory,
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
unnestingPositionWriterFactory, normalizedKeyComputerFactoryProvider,
expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer,
partialAggregationTypeComputer, predEvaluatorFactoryProvider,
physicalOptimizationConfig.getFrameSize(), clusterLocations, warningCollector,
- maxWarnings);
-=======
- normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer,
- oc, expressionEvalSizeComputer, partialAggregationTypeComputer,
- predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(),
- clusterLocations, warningCollector, maxWarnings, physicalOptimizationConfig);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
+ maxWarnings, physicalOptimizationConfig);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 03c7088..0f3823e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -26,7 +26,6 @@
public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final int SORT_SAMPLES_DEFAULT = 100;
public static final boolean SORT_PARALLEL_DEFAULT = true;
public static final boolean INDEX_ONLY_DEFAULT = true;
@@ -35,10 +34,6 @@
public static final boolean SUBPLAN_MERGE_DEFAULT = true;
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
-=======
- public static final int SORT_SAMPLES = 100;
public static final int EXTERNAL_SCAN_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(4, StorageUtil.StorageUnit.KILOBYTE);
- public static final boolean SORT_PARALLEL = true;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index cb56f9b..bfff925 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -39,16 +39,13 @@
private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
private static final String SORT_PARALLEL = "SORT_PARALLEL";
private static final String SORT_SAMPLES = "SORT_SAMPLES";
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private static final String INDEX_ONLY = "INDEX_ONLY";
private static final String SANITY_CHECK = "SANITY_CHECK";
private static final String EXTERNAL_FIELD_PUSHDOWN = "EXTERNAL_FIELD_PUSHDOWN";
private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
-=======
private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
private final Properties properties = new Properties();
@@ -182,7 +179,6 @@
setInt(SORT_SAMPLES, sortSamples);
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public void setIndexOnly(boolean indexOnly) {
setBoolean(INDEX_ONLY, indexOnly);
}
@@ -229,14 +225,14 @@
public void setMinMemoryAllocation(boolean value) {
setBoolean(MIN_MEMORY_ALLOCATION, value);
-=======
+ }
+
public int getExternalScanBufferSize() {
return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
}
public void setExternalScanBufferSize(int bufferSize) {
setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
private void setInt(String property, int value) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
Gerrit-Change-Number: 10926
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: merge mad-hatter into cheshire-cat
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Murtadha Hubail <mh...@apache.org>:
Hello Anon. E. Moose #1000171,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926
to look at the new patch set (#2).
Change subject: merge mad-hatter into cheshire-cat
......................................................................
merge mad-hatter into cheshire-cat
Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
D asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
26 files changed, 14 insertions(+), 438 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/26/10926/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
Gerrit-Change-Number: 10926
Gerrit-PatchSet: 2
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-CC: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-MessageType: newpatchset
Change in asterixdb[cheshire-cat]: merge mad-hatter into cheshire-cat
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926 )
Change subject: merge mad-hatter into cheshire-cat
......................................................................
merge mad-hatter into cheshire-cat
Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
R asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
D asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
26 files changed, 16 insertions(+), 433 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/26/10926/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index a091905..9683b9d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -130,12 +130,12 @@
public static final String PREFIX_INTERNAL_PARAMETERS = "_internal";
// A white list of supported configurable parameters.
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private static final Set<String> CONFIGURABLE_PARAMETER_NAMES = ImmutableSet.of(
CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
+ CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY,
CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
@@ -147,20 +147,6 @@
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK);
-=======
- private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
- ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY,
- CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
- CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY,
- CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
- CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
- FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
- StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
- FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
- SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
- AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION,
- DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.001.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.003.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.004.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.005.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.006.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/custom-buffer-size/external_dataset.999.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/custom-buffer-size/external_dataset.999.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.002.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.002.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.003.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.003.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.004.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.004.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.005.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.005.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/custom-buffer-size/external_dataset.006.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/custom-buffer-size/external_dataset.006.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
deleted file mode 100644
index f8e48d7..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ /dev/null
@@ -1,263 +0,0 @@
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
- <test-group name="external-dataset">
- <test-case FilePath="external-dataset/aws/s3/">
- <compilation-unit name="custom-buffer-size">
- <output-dir compare="Text">custom-buffer-size</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/json">
- <output-dir compare="Text">aws/s3/json/json</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/gz">
- <output-dir compare="Text">aws/s3/json/gz</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/json/mixed">
- <output-dir compare="Text">aws/s3/json/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/csv">
- <output-dir compare="Text">aws/s3/csv/csv</output-dir>
- </compilation-unit>
- </test-case><test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/gz">
- <output-dir compare="Text">aws/s3/csv/gz</output-dir>
- </compilation-unit>
- </test-case><test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/csv/mixed">
- <output-dir compare="Text">aws/s3/csv/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/tsv">
- <output-dir compare="Text">aws/s3/tsv/tsv</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/gz">
- <output-dir compare="Text">aws/s3/tsv/gz</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/tsv/mixed">
- <output-dir compare="Text">aws/s3/tsv/mixed</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/empty-string-definition">
- <output-dir compare="Text">aws/s3/empty-string-definition</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/over-1000-objects">
- <output-dir compare="Text">aws/s3/over-1000-objects</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/malformed-json">
- <output-dir compare="Text">aws/s3/malformed-json</output-dir>
- <expected-error>Parsing error at malformed-data/duplicate-fields.json line 1 field field: Duplicate field 'field'</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json.json line 1 field field: Unexpected character ('}' (code 125)): was expecting double-quote to start field name</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-json-2.json line 4 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token 'truee': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- <expected-error>Parsing error at malformed-data/malformed-jsonl-2.json line 11 field array_f: Unexpected character (']' (code 93)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false')</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/definition-does-not-exist">
- <output-dir compare="Text">aws/s3/definition-does-not-exist</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/invalid-endpoint">
- <output-dir compare="Text">aws/s3/invalid-endpoint</output-dir>
- <expected-error>External source error. Invalid service endpoint ^invalid-endpoint^</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/bucket-does-not-exist">
- <output-dir compare="Text">aws/s3/bucket-does-not-exist</output-dir>
- <expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null)</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/definition-points-to-nothing">
- <output-dir compare="Text">aws/s3/no-files-returned/definition-points-to-nothing</output-dir>
- <source-location>false</source-location>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/exclude-all-files">
- <output-dir compare="Text">aws/s3/no-files-returned/exclude-all-files</output-dir>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset" check-warnings="true">
- <compilation-unit name="aws/s3/no-files-returned/include-no-files">
- <output-dir compare="Text">aws/s3/no-files-returned/include-no-files</output-dir>
- <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/non-s3-region">
- <output-dir compare="Text">aws/s3/non-s3-region</output-dir>
- </compilation-unit>
- </test-case>
- </test-group>
- <test-group name="include-exclude">
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-1">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-1</output-dir>
- <expected-error>Invalid format for property "exclude1"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-2">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-2</output-dir>
- <expected-error>Invalid format for property "exclude#"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/bad-name-3">
- <output-dir compare="Text">aws/s3/include-exclude/bad-name-3</output-dir>
- <expected-error>Invalid format for property "exclude#hello"</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/both">
- <output-dir compare="Text">aws/s3/include-exclude/both</output-dir>
- <expected-error>The parameters "include" and "exclude" cannot be provided at the same time</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-all">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-all</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-1">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-1</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-2">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-2</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-3">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-3</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-4">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-4</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-5">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-5</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/exclude-6">
- <output-dir compare="Text">aws/s3/include-exclude/exclude-6</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-all">
- <output-dir compare="Text">aws/s3/include-exclude/include-all</output-dir>
- <expected-error>Malformed input stream</expected-error>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-1">
- <output-dir compare="Text">aws/s3/include-exclude/include-1</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-2">
- <output-dir compare="Text">aws/s3/include-exclude/include-2</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-3">
- <output-dir compare="Text">aws/s3/include-exclude/include-3</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-4">
- <output-dir compare="Text">aws/s3/include-exclude/include-4</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-5">
- <output-dir compare="Text">aws/s3/include-exclude/include-5</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-6">
- <output-dir compare="Text">aws/s3/include-exclude/include-6</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-7">
- <output-dir compare="Text">aws/s3/include-exclude/include-7</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-8">
- <output-dir compare="Text">aws/s3/include-exclude/include-8</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-9">
- <output-dir compare="Text">aws/s3/include-exclude/include-9</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-10">
- <output-dir compare="Text">aws/s3/include-exclude/include-10</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-11">
- <output-dir compare="Text">aws/s3/include-exclude/include-11</output-dir>
- </compilation-unit>
- </test-case>
- <test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/include-exclude/include-12">
- <output-dir compare="Text">aws/s3/include-exclude/include-12</output-dir>
- </compilation-unit>
- </test-case>
- </test-group>
-</test-suite>
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 765ccd0..60ecf7f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -130,14 +130,11 @@
public Object defaultValue() {
return defaultValue;
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
@Override
public boolean hidden() {
- return this == COMPILER_STRINGOFFSET || this == COMPILER_EXTERNALSCANMEMORY;
+ return this == COMPILER_EXTERNALSCANMEMORY;
}
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
public static final String COMPILER_SORTMEMORY_KEY = Option.COMPILER_SORTMEMORY.ini();
@@ -156,7 +153,6 @@
public static final String COMPILER_SORT_SAMPLES_KEY = Option.COMPILER_SORT_SAMPLES.ini();
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final String COMPILER_INDEXONLY_KEY = Option.COMPILER_INDEXONLY.ini();
public static final String COMPILER_INTERNAL_SANITYCHECK_KEY = Option.COMPILER_INTERNAL_SANITYCHECK.ini();
@@ -168,9 +164,8 @@
public static final String COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY = Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN.ini();
public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini();
-=======
+
public static final String COMPILER_EXTERNALSCANMEMORY_KEY = Option.COMPILER_EXTERNALSCANMEMORY.ini();
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
public static final int COMPILER_PARALLELISM_AS_STORAGE = 0;
@@ -214,7 +209,6 @@
return accessor.getInt(Option.COMPILER_SORT_SAMPLES);
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public boolean isIndexOnly() {
return accessor.getBoolean(Option.COMPILER_INDEXONLY);
}
@@ -237,9 +231,9 @@
public boolean getMinMemoryAllocation() {
return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION);
-=======
+ }
+
public int getExternalScanMemorySize() {
return accessor.getInt(Option.COMPILER_EXTERNALSCANMEMORY);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index cb0e799..c4dbd85 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -58,7 +58,6 @@
compilerProperties.getWindowMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_WINDOW, sourceLoc);
int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc);
int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
boolean fullParallelSort = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
compilerProperties.getSortParallel());
boolean indexOnly = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_INDEXONLY_KEY,
@@ -73,13 +72,9 @@
CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown());
boolean minMemoryAllocation = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation());
-=======
- boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig);
int externalScanBufferSize = getExternalScanBufferSize(
(String) querySpecificConfig.get(CompilerProperties.COMPILER_EXTERNALSCANMEMORY_KEY),
compilerProperties.getExternalScanMemorySize(), sourceLoc);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig();
physOptConf.setFrameSize(frameSize);
physOptConf.setMaxFramesExternalSort(sortFrameLimit);
@@ -89,17 +84,13 @@
physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit);
physOptConf.setSortParallel(fullParallelSort);
physOptConf.setSortSamples(sortNumSamples);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
physOptConf.setIndexOnly(indexOnly);
physOptConf.setSanityCheckEnabled(sanityCheck);
physOptConf.setExternalFieldPushdown(externalFieldPushdown);
physOptConf.setSubplanMerge(subplanMerge);
physOptConf.setSubplanNestedPushdown(subplanNestedPushdown);
physOptConf.setMinMemoryAllocation(minMemoryAllocation);
-=======
physOptConf.setExternalScanBufferSize(externalScanBufferSize);
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
return physOptConf;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index acce233..54a2987 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
-import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -28,73 +26,37 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-=======
-import org.apache.asterix.external.input.stream.AbstractMultipleInputStream;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public class AwsS3InputStream extends AbstractExternalInputStream {
-=======
-public class AwsS3InputStream extends AbstractMultipleInputStream {
private static final Logger LOGGER = LogManager.getLogger();
// Configuration
- private final String bucket;
private final int bufferSize;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
private final S3Client s3Client;
private final String bucket;
public AwsS3InputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
super(configuration, filePaths);
-=======
- this.filePaths = filePaths;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
this.s3Client = buildAwsS3Client(configuration);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
this.bucket = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-=======
this.bufferSize = ExternalDataUtils.getOrDefaultBufferSize(configuration);
- this.bucket = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
@Override
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
protected boolean getInputStream() throws IOException {
-=======
- protected boolean advance() throws IOException {
- // No files to read for this partition
- if (filePaths == null || filePaths.isEmpty()) {
- return false;
- }
-
- // Finished reading all the files
- if (nextFileIndex >= filePaths.size()) {
- return false;
- }
-
- // Close the current stream before going to the next one
- if (in != null) {
- CleanupUtils.close(in, null);
- }
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
String fileName = filePaths.get(nextFileIndex);
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(fileName).build();
@@ -112,21 +74,10 @@
}
// Use gzip stream if needed
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
String lowerCaseFileName = fileName.toLowerCase();
if (lowerCaseFileName.endsWith(".gz") || lowerCaseFileName.endsWith(".gzip")) {
- in = new GZIPInputStream(in, ExternalDataConstants.DEFAULT_BUFFER_SIZE);
-=======
- if (StringUtils.endsWithIgnoreCase(fileName, ".gz") || StringUtils.endsWithIgnoreCase(fileName, ".gzip")) {
in = new GZIPInputStream(in, bufferSize);
}
-
- // Current file ready, point to the next file
- nextFileIndex++;
- if (notificationHandler != null) {
- notificationHandler.notifyNewSource();
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
- }
return true;
}
@@ -137,41 +88,4 @@
throw HyracksDataException.create(ex);
}
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
-=======
-
- @Override
- public boolean stop() {
- return false;
- }
-
- @Override
- public boolean handleException(Throwable th) {
- return false;
- }
-
- @Override
- public void close() throws IOException {
- if (in != null) {
- CleanupUtils.close(in, null);
- }
- if (s3Client != null) {
- CleanupUtils.close(s3Client, null);
- }
- }
-
- @Override
- public String getStreamName() {
- return getStreamNameAt(nextFileIndex - 1);
- }
-
- @Override
- public String getPreviousStreamName() {
- return getStreamNameAt(nextFileIndex - 2);
- }
-
- private String getStreamNameAt(int fileIndex) {
- return fileIndex < 0 || filePaths == null || filePaths.isEmpty() ? "" : filePaths.get(fileIndex);
- }
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
index 8c4488d..0098778 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/AsterixInputStreamReader.java
@@ -27,26 +27,18 @@
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.FeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public class AsterixInputStreamReader extends Reader {
private final AsterixInputStream in;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private final CharsetDecoder decoder;
- private final byte[] bytes;
- protected final ByteBuffer byteBuffer;
- protected final CharBuffer charBuffer;
+ private byte[] bytes;
+ protected ByteBuffer byteBuffer;
+ protected CharBuffer charBuffer;
protected boolean done = false;
protected boolean remaining = false;
-=======
- private final byte[] bytes;
- private final ByteBuffer byteBuffer;
- private final CharBuffer charBuffer;
- private final CharsetDecoder decoder;
- private boolean done = false;
- private boolean remaining = false;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
public AsterixInputStreamReader(AsterixInputStream in, int bufferSize) {
this.in = in;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index a415ddb..088cd58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -119,11 +119,8 @@
// a string representing the NULL value
public static final String KEY_NULL_STR = "null";
public static final String KEY_REDACT_WARNINGS = "redact-warnings";
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final String KEY_REQUESTED_FIELDS = "requested-fields";
-=======
public static final String KEY_EXTERNAL_SCAN_BUFFER_SIZE = "external-scan-buffer-size";
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
/**
* Keys for adapter name
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index c89b669..dffde9c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -18,12 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
import java.util.HashMap;
-=======
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
-
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
import java.util.List;
import java.util.Map;
@@ -115,21 +111,14 @@
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
- Map<String, String> properties = addProjectionInfo(projectionInfo, edd.getProperties());
- ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
- edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
- tupleFilterFactory, outputLimit);
-=======
PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
Map<String, String> properties = edd.getProperties();
properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
- return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
+ return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
+ tupleFilterFactory, outputLimit);
case INTERNAL:
DataSourceId id = getId();
DataverseName dataverseName = id.getDataverseName();
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index ba6f96b..c6a79a5 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -102,19 +102,11 @@
serializerDeserializerProvider, hashFunctionFactoryProvider, hashFunctionFamilyProvider,
comparatorFactoryProvider, typeTraitProvider, binaryBooleanInspectorFactory,
binaryIntegerInspectorFactory, printerProvider, missingWriterFactory,
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
unnestingPositionWriterFactory, normalizedKeyComputerFactoryProvider,
expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer,
partialAggregationTypeComputer, predEvaluatorFactoryProvider,
physicalOptimizationConfig.getFrameSize(), clusterLocations, warningCollector,
- maxWarnings);
-=======
- normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer,
- oc, expressionEvalSizeComputer, partialAggregationTypeComputer,
- predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(),
- clusterLocations, warningCollector, maxWarnings, physicalOptimizationConfig);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
-
+ maxWarnings, physicalOptimizationConfig);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 03c7088..0f3823e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -26,7 +26,6 @@
public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks";
public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME);
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public static final int SORT_SAMPLES_DEFAULT = 100;
public static final boolean SORT_PARALLEL_DEFAULT = true;
public static final boolean INDEX_ONLY_DEFAULT = true;
@@ -35,10 +34,6 @@
public static final boolean SUBPLAN_MERGE_DEFAULT = true;
public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true;
public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true;
-=======
- public static final int SORT_SAMPLES = 100;
public static final int EXTERNAL_SCAN_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(4, StorageUtil.StorageUnit.KILOBYTE);
- public static final boolean SORT_PARALLEL = true;
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
index cb56f9b..bfff925 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -39,16 +39,13 @@
private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
private static final String SORT_PARALLEL = "SORT_PARALLEL";
private static final String SORT_SAMPLES = "SORT_SAMPLES";
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
private static final String INDEX_ONLY = "INDEX_ONLY";
private static final String SANITY_CHECK = "SANITY_CHECK";
private static final String EXTERNAL_FIELD_PUSHDOWN = "EXTERNAL_FIELD_PUSHDOWN";
private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE";
private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN";
private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION";
-=======
private static final String EXTERNAL_SCAN_BUFFER_SIZE = "EXTERNAL_SCAN_BUFFER_SIZE";
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
private final Properties properties = new Properties();
@@ -182,7 +179,6 @@
setInt(SORT_SAMPLES, sortSamples);
}
-<<<<<<< HEAD (244c3c [NO ISSUE][HYR][UTIL] Abort retries on expired span no retry)
public void setIndexOnly(boolean indexOnly) {
setBoolean(INDEX_ONLY, indexOnly);
}
@@ -229,14 +225,14 @@
public void setMinMemoryAllocation(boolean value) {
setBoolean(MIN_MEMORY_ALLOCATION, value);
-=======
+ }
+
public int getExternalScanBufferSize() {
return getInt(EXTERNAL_SCAN_BUFFER_SIZE, AlgebricksConfig.EXTERNAL_SCAN_BUFFER_SIZE);
}
public void setExternalScanBufferSize(int bufferSize) {
setInt(EXTERNAL_SCAN_BUFFER_SIZE, bufferSize);
->>>>>>> BRANCH (3ea8ef [ASTERIXDB-2870][EXT]: Close client after it is used)
}
private void setInt(String property, int value) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10926
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I30e7a87937982c16a347464dc0efa069a84f9096
Gerrit-Change-Number: 10926
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-MessageType: newchange