You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/06/26 00:02:03 UTC
[asterixdb] branch master updated (0f096f3 -> b2351df)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 0f096f3 [ASTERIXDB-2752][MTD] Add metadata for synonyms on upgrade
new 722db20 [ASTERIXDB-2751][EXT] Issue a warning if external dataset configuration returns no files
new c25c36d [NO ISSUE] Pass item type when creating external dataset properties
new 7ac75e7 Merge commit 'c25c36d1b0' from 'gerrit/mad-hatter'
new b2351df Merge "Merge commit 'c25c36d1b0' from 'gerrit/mad-hatter'"
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../optimizer/rules/ConstantFoldingRule.java | 4 ++-
.../asterix/app/translator/QueryTranslator.java | 18 +++++-----
.../apache/asterix/test/common/TestExecutor.java | 1 -
.../test.000.ddl.sqlpp} | 35 ++++++++++----------
.../test.001.query.sqlpp} | 22 ++-----------
.../test.099.ddl.sqlpp | 0
.../exclude-all-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../exclude-all-files/test.001.query.sqlpp} | 22 ++-----------
.../exclude-all-files}/test.099.ddl.sqlpp | 0
.../include-no-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../include-no-files/test.001.query.sqlpp} | 22 ++-----------
.../include-no-files}/test.099.ddl.sqlpp | 0
.../definition-points-to-nothing}/result.001.adm | 0
.../exclude-all-files}/result.001.adm | 0
.../include-no-files}/result.001.adm | 0
.../runtimets/testsuite_external_dataset.xml | 18 ++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 ++++++-------
.../asterix/common/exceptions/ErrorCode.java | 15 +++++----
.../common/exceptions/NoOpWarningCollector.java} | 28 ++++++++++------
.../asterix/common/external/IAdapterFactory.java | 6 ++--
.../src/main/resources/asx_errormsg/en.properties | 15 +++++----
.../adapter/factory/GenericAdapterFactory.java | 15 +++++----
.../adapter/factory/LookupAdapterFactory.java | 10 +++---
.../external/api/IExternalDataSourceFactory.java | 3 +-
.../external/input/HDFSDataSourceFactory.java | 4 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 17 +++++++---
.../record/reader/aws/AwsS3ReaderFactory.java | 5 +--
.../reader/hdfs/HDFSLookupReaderFactory.java | 4 ++-
.../reader/http/HttpServerRecordReaderFactory.java | 4 ++-
.../record/reader/rss/RSSRecordReaderFactory.java | 4 ++-
.../reader/stream/StreamRecordReaderFactory.java | 7 ++--
.../reader/twitter/TwitterRecordReaderFactory.java | 4 ++-
.../stream/factory/LocalFSInputStreamFactory.java | 4 ++-
.../factory/SocketClientInputStreamFactory.java | 4 ++-
.../factory/SocketServerInputStreamFactory.java | 4 ++-
.../factory/TwitterFirehoseStreamFactory.java | 4 ++-
.../operators/FeedIntakeOperatorDescriptor.java | 3 +-
.../external/provider/AdapterFactoryProvider.java | 15 +++++----
.../provider/LookupReaderFactoryProvider.java | 5 +--
.../asterix/external/util/ExternalDataUtils.java | 38 ++++++++++++++++++----
.../apache/asterix/external/util/ParseUtil.java | 5 +--
.../reader/RecordWithPKTestReaderFactory.java | 4 ++-
.../record/reader/kv/KVTestReaderFactory.java | 4 ++-
.../library/adapter/TestTypedAdapterFactory.java | 4 ++-
.../metadata/declared/DatasetDataSource.java | 5 +--
.../declared/FunctionDataSourceFactory.java | 3 +-
.../metadata/declared/LoadableDataSource.java | 2 +-
.../metadata/declared/MetadataProvider.java | 11 ++++---
.../asterix/metadata/feeds/FeedMetadataUtil.java | 12 ++++---
.../metadata/utils/ExternalIndexingOperations.java | 3 +-
.../api/HeuristicCompilerFactoryBuilder.java | 2 +-
.../algebricks/core/jobgen/impl/JobGenContext.java | 10 +++++-
.../hyracks/api/exceptions/SourceLocation.java | 18 +++++++---
.../org/apache/hyracks/api/exceptions/Warning.java | 3 +-
.../file/FieldCursorForDelimitedDataParser.java | 4 +--
55 files changed, 305 insertions(+), 237 deletions(-)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp} (53%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp} (53%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => no-files-returned/definition-points-to-nothing}/test.099.ddl.sqlpp (100%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp} (53%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp} (53%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => no-files-returned/exclude-all-files}/test.099.ddl.sqlpp (100%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp} (53%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp} (53%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => no-files-returned/include-no-files}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/{include-exclude/exclude-all => no-files-returned/definition-points-to-nothing}/result.001.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/{include-exclude/exclude-all => no-files-returned/exclude-all-files}/result.001.adm (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/{include-exclude/exclude-all => no-files-returned/include-no-files}/result.001.adm (100%)
copy asterixdb/{asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java => asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java} (61%)
[asterixdb] 04/04: Merge "Merge commit 'c25c36d1b0' from
'gerrit/mad-hatter'"
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b2351dfeebac9eb6cee4edb8cfe067a1532c9fbd
Merge: 0f096f3 7ac75e7
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Jun 25 23:58:49 2020 +0000
Merge "Merge commit 'c25c36d1b0' from 'gerrit/mad-hatter'"
.../optimizer/rules/ConstantFoldingRule.java | 4 ++-
.../asterix/app/translator/QueryTranslator.java | 18 +++++-----
.../apache/asterix/test/common/TestExecutor.java | 1 -
.../test.000.ddl.sqlpp} | 35 ++++++++++----------
.../test.001.query.sqlpp} | 22 ++-----------
.../test.099.ddl.sqlpp} | 20 +-----------
.../exclude-all-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../exclude-all-files/test.001.query.sqlpp} | 22 ++-----------
.../exclude-all-files/test.099.ddl.sqlpp} | 20 +-----------
.../include-no-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../include-no-files/test.001.query.sqlpp} | 22 ++-----------
.../include-no-files/test.099.ddl.sqlpp} | 20 +-----------
.../definition-points-to-nothing/result.001.adm | 1 +
.../exclude-all-files/result.001.adm | 1 +
.../include-no-files/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 18 ++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 ++++++-------
.../asterix/common/exceptions/ErrorCode.java | 15 +++++----
.../common/exceptions/NoOpWarningCollector.java} | 28 ++++++++++------
.../asterix/common/external/IAdapterFactory.java | 6 ++--
.../src/main/resources/asx_errormsg/en.properties | 15 +++++----
.../adapter/factory/GenericAdapterFactory.java | 15 +++++----
.../adapter/factory/LookupAdapterFactory.java | 10 +++---
.../external/api/IExternalDataSourceFactory.java | 3 +-
.../external/input/HDFSDataSourceFactory.java | 4 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 17 +++++++---
.../record/reader/aws/AwsS3ReaderFactory.java | 5 +--
.../reader/hdfs/HDFSLookupReaderFactory.java | 4 ++-
.../reader/http/HttpServerRecordReaderFactory.java | 4 ++-
.../record/reader/rss/RSSRecordReaderFactory.java | 4 ++-
.../reader/stream/StreamRecordReaderFactory.java | 7 ++--
.../reader/twitter/TwitterRecordReaderFactory.java | 4 ++-
.../stream/factory/LocalFSInputStreamFactory.java | 4 ++-
.../factory/SocketClientInputStreamFactory.java | 4 ++-
.../factory/SocketServerInputStreamFactory.java | 4 ++-
.../factory/TwitterFirehoseStreamFactory.java | 4 ++-
.../operators/FeedIntakeOperatorDescriptor.java | 3 +-
.../external/provider/AdapterFactoryProvider.java | 15 +++++----
.../provider/LookupReaderFactoryProvider.java | 5 +--
.../asterix/external/util/ExternalDataUtils.java | 38 ++++++++++++++++++----
.../apache/asterix/external/util/ParseUtil.java | 5 +--
.../reader/RecordWithPKTestReaderFactory.java | 4 ++-
.../record/reader/kv/KVTestReaderFactory.java | 4 ++-
.../library/adapter/TestTypedAdapterFactory.java | 4 ++-
.../metadata/declared/DatasetDataSource.java | 5 +--
.../declared/FunctionDataSourceFactory.java | 3 +-
.../metadata/declared/LoadableDataSource.java | 2 +-
.../metadata/declared/MetadataProvider.java | 11 ++++---
.../asterix/metadata/feeds/FeedMetadataUtil.java | 12 ++++---
.../metadata/utils/ExternalIndexingOperations.java | 3 +-
.../api/HeuristicCompilerFactoryBuilder.java | 2 +-
.../algebricks/core/jobgen/impl/JobGenContext.java | 10 +++++-
.../hyracks/api/exceptions/SourceLocation.java | 18 +++++++---
.../org/apache/hyracks/api/exceptions/Warning.java | 3 +-
.../file/FieldCursorForDelimitedDataParser.java | 4 +--
55 files changed, 311 insertions(+), 294 deletions(-)
[asterixdb] 02/04: [NO ISSUE] Pass item type when creating external
dataset properties
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c25c36d1b08a6c3cc56c912faf2414a3f754934a
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Tue Jun 23 15:20:15 2020 +0300
[NO ISSUE] Pass item type when creating external dataset properties
Change-Id: I8b6693d229a36358eda584ddfb95d1819f76ce5c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6963
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../org/apache/asterix/app/translator/QueryTranslator.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ab2cf61..0f669ef 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -737,12 +737,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
break;
case EXTERNAL:
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
- Map<String, String> properties =
- createExternalDatasetProperties(dataverseName, dd, metadataProvider, mdTxnCtx);
+ Map<String, String> properties = createExternalDatasetProperties(dataverseName, dd, itemTypeEntity,
+ metadataProvider, mdTxnCtx);
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
- validateExternalDatasetProperties(externalDetails, properties, itemTypeEntity,
- dd.getSourceLocation(), mdTxnCtx);
+ validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx);
datasetDetails = new ExternalDatasetDetails(externalDetails.getAdapter(), properties, new Date(),
TransactionState.COMMIT);
break;
@@ -853,7 +852,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
protected Map<String, String> createExternalDatasetProperties(String dataverseName, DatasetDecl dd,
- MetadataProvider metadataProvider, MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
+ Datatype itemType, MetadataProvider metadataProvider, MetadataTransactionContext mdTxnCtx)
+ throws AlgebricksException {
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
return externalDetails.getProperties();
}
@@ -3240,8 +3240,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
- Map<String, String> properties, Datatype itemType, SourceLocation srcLoc,
- MetadataTransactionContext mdTxnCtx) throws AlgebricksException, HyracksDataException {
+ Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx)
+ throws AlgebricksException, HyracksDataException {
// Validate adapter specific properties
String adapter = externalDetails.getAdapter();
Map<String, String> details = new HashMap<>(properties);
[asterixdb] 03/04: Merge commit 'c25c36d1b0' from
'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7ac75e7ba9966fa11e794bd839f22e5a6669c45a
Merge: 44809d7 c25c36d
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Wed Jun 24 19:12:32 2020 +0300
Merge commit 'c25c36d1b0' from 'gerrit/mad-hatter'
Change-Id: I8a78950cb81f0a29ceae94fa439eec700c17d06d
.../optimizer/rules/ConstantFoldingRule.java | 4 ++-
.../asterix/app/translator/QueryTranslator.java | 18 +++++-----
.../apache/asterix/test/common/TestExecutor.java | 1 -
.../test.000.ddl.sqlpp} | 35 ++++++++++----------
.../test.001.query.sqlpp} | 22 ++-----------
.../test.099.ddl.sqlpp} | 20 +-----------
.../exclude-all-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../exclude-all-files/test.001.query.sqlpp} | 22 ++-----------
.../exclude-all-files/test.099.ddl.sqlpp} | 20 +-----------
.../include-no-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../include-no-files/test.001.query.sqlpp} | 22 ++-----------
.../include-no-files/test.099.ddl.sqlpp} | 20 +-----------
.../definition-points-to-nothing/result.001.adm | 1 +
.../exclude-all-files/result.001.adm | 1 +
.../include-no-files/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 18 ++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 ++++++-------
.../asterix/common/exceptions/ErrorCode.java | 15 +++++----
.../common/exceptions/NoOpWarningCollector.java} | 28 ++++++++++------
.../asterix/common/external/IAdapterFactory.java | 6 ++--
.../src/main/resources/asx_errormsg/en.properties | 15 +++++----
.../adapter/factory/GenericAdapterFactory.java | 15 +++++----
.../adapter/factory/LookupAdapterFactory.java | 10 +++---
.../external/api/IExternalDataSourceFactory.java | 3 +-
.../external/input/HDFSDataSourceFactory.java | 4 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 17 +++++++---
.../record/reader/aws/AwsS3ReaderFactory.java | 5 +--
.../reader/hdfs/HDFSLookupReaderFactory.java | 4 ++-
.../reader/http/HttpServerRecordReaderFactory.java | 4 ++-
.../record/reader/rss/RSSRecordReaderFactory.java | 4 ++-
.../reader/stream/StreamRecordReaderFactory.java | 7 ++--
.../reader/twitter/TwitterRecordReaderFactory.java | 4 ++-
.../stream/factory/LocalFSInputStreamFactory.java | 4 ++-
.../factory/SocketClientInputStreamFactory.java | 4 ++-
.../factory/SocketServerInputStreamFactory.java | 4 ++-
.../factory/TwitterFirehoseStreamFactory.java | 4 ++-
.../operators/FeedIntakeOperatorDescriptor.java | 3 +-
.../external/provider/AdapterFactoryProvider.java | 15 +++++----
.../provider/LookupReaderFactoryProvider.java | 5 +--
.../asterix/external/util/ExternalDataUtils.java | 38 ++++++++++++++++++----
.../apache/asterix/external/util/ParseUtil.java | 5 +--
.../reader/RecordWithPKTestReaderFactory.java | 4 ++-
.../record/reader/kv/KVTestReaderFactory.java | 4 ++-
.../library/adapter/TestTypedAdapterFactory.java | 4 ++-
.../metadata/declared/DatasetDataSource.java | 5 +--
.../declared/FunctionDataSourceFactory.java | 3 +-
.../metadata/declared/LoadableDataSource.java | 2 +-
.../metadata/declared/MetadataProvider.java | 11 ++++---
.../asterix/metadata/feeds/FeedMetadataUtil.java | 12 ++++---
.../metadata/utils/ExternalIndexingOperations.java | 3 +-
.../api/HeuristicCompilerFactoryBuilder.java | 2 +-
.../algebricks/core/jobgen/impl/JobGenContext.java | 10 +++++-
.../hyracks/api/exceptions/SourceLocation.java | 18 +++++++---
.../org/apache/hyracks/api/exceptions/Warning.java | 3 +-
.../file/FieldCursorForDelimitedDataParser.java | 4 +--
55 files changed, 311 insertions(+), 294 deletions(-)
diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 2b703a2,494b1b4..4ca3275
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@@ -29,9 -29,9 +29,10 @@@ import org.apache.asterix.common.config
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+ import org.apache.asterix.common.exceptions.NoOpWarningCollector;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.formats.nontagged.ADMPrinterFactoryProvider;
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index a5fef11,0f669ef..d1c5aeb
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -882,8 -851,9 +881,9 @@@ public class QueryTranslator extends Ab
}
}
- protected Map<String, String> createExternalDatasetProperties(String dataverseName, DatasetDecl dd,
+ protected Map<String, String> createExternalDatasetProperties(DataverseName dataverseName, DatasetDecl dd,
- MetadataProvider metadataProvider, MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
+ Datatype itemType, MetadataProvider metadataProvider, MetadataTransactionContext mdTxnCtx)
+ throws AlgebricksException {
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
return externalDetails.getProperties();
}
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index e1f3084,63773dd..1b9c610
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@@ -2862,23 -2837,6 +2862,23 @@@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="aggregate-subclause">
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_01">
+ <output-dir compare="Text">agg_filter_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_02_neg">
+ <output-dir compare="Text">agg_filter_01</output-dir>
- <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
- <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier r</expected-error>
- <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
- <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1121: Illegal use of aggregate FILTER clause</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="array_fun">
<test-case FilePath="array_fun">
<compilation-unit name="array_prepend">
@@@ -5612,29 -5406,6 +5612,29 @@@
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-1">
+ <output-dir compare="Text">grouping-sets-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-2">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-3-negative">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
- <expected-error>ASX1119: Unexpected alias: v21</expected-error>
- <expected-error>ASX1119: Unexpected alias: v22</expected-error>
- <expected-error>ASX1119: Unexpected alias: v23</expected-error>
++ <expected-error>ASX1120: Unexpected alias: v21</expected-error>
++ <expected-error>ASX1120: Unexpected alias: v22</expected-error>
++ <expected-error>ASX1120: Unexpected alias: v23</expected-error>
+ <expected-error>ASX1087: Invalid number of arguments for function grouping</expected-error>
- <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1117: Too many grouping sets in group by clause: 512. Maximum allowed: 128.</expected-error>
++ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1119: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1118: Too many grouping sets in group by clause: 512. Maximum allowed: 128.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
<compilation-unit name="having">
<output-dir compare="Text">core-02</output-dir>
</compilation-unit>
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 503ff6e,d57c433..fdba8d6
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@@ -203,13 -202,7 +203,14 @@@ public class ErrorCode
public static final int PROPERTY_INVALID_VALUE_TYPE = 1111;
public static final int INVALID_PROPERTY_FORMAT = 1112;
public static final int INVALID_REGEX_PATTERN = 1113;
- public static final int SYNONYM_EXISTS = 1114;
- public static final int UNKNOWN_SYNONYM = 1115;
- public static final int UNKNOWN_LIBRARY = 1116;
- public static final int COMPILATION_GROUPING_SETS_OVERFLOW = 1117;
- public static final int COMPILATION_GROUPING_OPERATION_INVALID_ARG = 1118;
- public static final int COMPILATION_UNEXPECTED_ALIAS = 1119;
- public static final int COMPILATION_ILLEGAL_USE_OF_FILTER_CLAUSE = 1120;
+ public static final int EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES = 1114;
++ public static final int SYNONYM_EXISTS = 1115;
++ public static final int UNKNOWN_SYNONYM = 1116;
++ public static final int UNKNOWN_LIBRARY = 1117;
++ public static final int COMPILATION_GROUPING_SETS_OVERFLOW = 1118;
++ public static final int COMPILATION_GROUPING_OPERATION_INVALID_ARG = 1119;
++ public static final int COMPILATION_UNEXPECTED_ALIAS = 1120;
++ public static final int COMPILATION_ILLEGAL_USE_OF_FILTER_CLAUSE = 1121;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index e612503,5cf3968..9bbebf2
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@@ -198,13 -197,7 +198,14 @@@
1111 = Property \"%1$s\" expects value(s) of type %2$s
1112 = Invalid format for property \"%1$s\"
1113 = Invalid pattern %1$s
- 1114 = A synonym with this name %1$s already exists
- 1115 = Cannot find synonym with name %1$s
- 1116 = Unknown library %1$s
- 1117 = Too many grouping sets in group by clause: %1$s. Maximum allowed: %2$s.
- 1118 = Invalid argument to grouping() function
- 1119 = Unexpected alias: %1$s
- 1120 = Illegal use of aggregate FILTER clause
+ 1114 = The provided external dataset configuration returned no files from the external source
++1115 = A synonym with this name %1$s already exists
++1116 = Cannot find synonym with name %1$s
++1117 = Unknown library %1$s
++1118 = Too many grouping sets in group by clause: %1$s. Maximum allowed: %2$s.
++1119 = Invalid argument to grouping() function
++1120 = Unexpected alias: %1$s
++1121 = Illegal use of aggregate FILTER clause
# Feed Errors
3001 = Illegal state.
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index f0c6422,1300ac3..052a8e0
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@@ -37,10 -37,8 +37,11 @@@ import org.apache.asterix.common.except
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+ import org.apache.asterix.common.exceptions.WarningUtil;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 3107a00,c5109b7..3f5d0b0
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@@ -823,11 -782,13 +824,13 @@@ public class MetadataProvider implement
}
protected ITypedAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
- Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException {
+ Map<String, String> configuration, ARecordType itemType, ARecordType metaType,
+ IWarningCollector warningCollector) throws AlgebricksException {
try {
- configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName());
+ configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName().getCanonicalForm());
- ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(
- getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType);
+ ITypedAdapterFactory adapterFactory =
+ AdapterFactoryProvider.getAdapterFactory(getApplicationContext().getServiceContext(), adapterName,
+ configuration, itemType, metaType, warningCollector);
// check to see if dataset is indexed
Index filesIndex =
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index e0a8e83,792501b..24301e0
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@@ -28,11 -28,9 +28,12 @@@ import org.apache.asterix.common.except
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.MetadataException;
+ import org.apache.asterix.common.exceptions.NoOpWarningCollector;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
@@@ -56,7 -53,7 +57,8 @@@ import org.apache.hyracks.algebricks.co
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+ import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* A utility class for providing helper functions for feeds TODO: Refactor this
[asterixdb] 01/04: [ASTERIXDB-2751][EXT] Issue a warning if
external dataset configuration returns no files
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 722db2097c2ad4e39b9bffc0d615df1f6e25af90
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Tue Jun 23 03:33:39 2020 +0300
[ASTERIXDB-2751][EXT] Issue a warning if external dataset configuration returns no files
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Issue a warning when creating an external dataset if the dataset
configuration returns no files.
- Issue a warning when querying an external dataset if no files
are returned.
Change-Id: I0b2be49bf3c69c3e055d942a5c3f0106b3069d52
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6843
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../optimizer/rules/ConstantFoldingRule.java | 4 ++-
.../asterix/app/translator/QueryTranslator.java | 4 +--
.../apache/asterix/test/common/TestExecutor.java | 1 -
.../test.000.ddl.sqlpp} | 35 ++++++++++----------
.../test.001.query.sqlpp} | 22 ++-----------
.../test.099.ddl.sqlpp} | 20 +-----------
.../exclude-all-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../exclude-all-files/test.001.query.sqlpp} | 22 ++-----------
.../exclude-all-files/test.099.ddl.sqlpp} | 20 +-----------
.../include-no-files/test.000.ddl.sqlpp} | 35 ++++++++++----------
.../include-no-files/test.001.query.sqlpp} | 22 ++-----------
.../include-no-files/test.099.ddl.sqlpp} | 20 +-----------
.../definition-points-to-nothing/result.001.adm | 1 +
.../exclude-all-files/result.001.adm | 1 +
.../include-no-files/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 18 ++++++++++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../common/exceptions/NoOpWarningCollector.java} | 28 ++++++++++------
.../asterix/common/external/IAdapterFactory.java | 6 ++--
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../adapter/factory/GenericAdapterFactory.java | 15 +++++----
.../adapter/factory/LookupAdapterFactory.java | 10 +++---
.../external/api/IExternalDataSourceFactory.java | 3 +-
.../external/input/HDFSDataSourceFactory.java | 4 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 17 +++++++---
.../record/reader/aws/AwsS3ReaderFactory.java | 5 +--
.../reader/hdfs/HDFSLookupReaderFactory.java | 4 ++-
.../reader/http/HttpServerRecordReaderFactory.java | 4 ++-
.../record/reader/rss/RSSRecordReaderFactory.java | 4 ++-
.../reader/stream/StreamRecordReaderFactory.java | 7 ++--
.../reader/twitter/TwitterRecordReaderFactory.java | 4 ++-
.../stream/factory/LocalFSInputStreamFactory.java | 4 ++-
.../factory/SocketClientInputStreamFactory.java | 4 ++-
.../factory/SocketServerInputStreamFactory.java | 4 ++-
.../factory/TwitterFirehoseStreamFactory.java | 4 ++-
.../operators/FeedIntakeOperatorDescriptor.java | 3 +-
.../external/provider/AdapterFactoryProvider.java | 15 +++++----
.../provider/LookupReaderFactoryProvider.java | 5 +--
.../asterix/external/util/ExternalDataUtils.java | 38 ++++++++++++++++++----
.../apache/asterix/external/util/ParseUtil.java | 5 +--
.../reader/RecordWithPKTestReaderFactory.java | 4 ++-
.../record/reader/kv/KVTestReaderFactory.java | 4 ++-
.../library/adapter/TestTypedAdapterFactory.java | 4 ++-
.../metadata/declared/DatasetDataSource.java | 5 +--
.../declared/FunctionDataSourceFactory.java | 3 +-
.../metadata/declared/LoadableDataSource.java | 2 +-
.../metadata/declared/MetadataProvider.java | 11 ++++---
.../asterix/metadata/feeds/FeedMetadataUtil.java | 12 ++++---
.../metadata/utils/ExternalIndexingOperations.java | 3 +-
.../api/HeuristicCompilerFactoryBuilder.java | 2 +-
.../algebricks/core/jobgen/impl/JobGenContext.java | 10 +++++-
.../hyracks/api/exceptions/SourceLocation.java | 18 +++++++---
.../org/apache/hyracks/api/exceptions/Warning.java | 3 +-
.../file/FieldCursorForDelimitedDataParser.java | 4 +--
54 files changed, 279 insertions(+), 262 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index d8fa3d6..494b1b4 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -29,6 +29,7 @@ import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.NoOpWarningCollector;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer;
@@ -143,7 +144,8 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule {
BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY,
BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null,
new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
- ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null, 0);
+ ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null,
+ NoOpWarningCollector.INSTANCE, 0);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index f6536ac..ab2cf61 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2232,7 +2232,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
ExternalDataUtils.normalize(configuration);
ExternalDataUtils.validate(configuration);
feed = new Feed(dataverseName, feedName, configuration);
- FeedMetadataUtil.validateFeed(feed, mdTxnCtx, appCtx);
+ FeedMetadataUtil.validateFeed(feed, mdTxnCtx, appCtx, warningCollector);
MetadataManager.INSTANCE.addFeed(metadataProvider.getMetadataTxnContext(), feed);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
@@ -3256,6 +3256,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
*/
protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
throws CompilationException {
- ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc);
+ ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc, warningCollector);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 6bae914..5d71a26 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -1808,7 +1808,6 @@ public class TestExecutor {
List<CompilationUnit> cUnits = testCaseCtx.getTestCase().getCompilationUnit();
for (CompilationUnit cUnit : cUnits) {
List<String> expectedErrors = cUnit.getExpectedError();
- int expectedWarnCount = cUnit.getExpectedWarn().size();
BitSet expectedWarnings = new BitSet(cUnit.getExpectedWarn().size());
expectedWarnings.set(0, cUnit.getExpectedWarn().size());
LOGGER.info(
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp
index 598d9ff..b51a33f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.000.ddl.sqlpp
@@ -16,23 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
+drop dataverse test if exists;
+create dataverse test;
+use test;
-public class ParseUtil {
+drop type test if exists;
+create type test as open {
+};
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="include-exclude"),
+("definition"="point-to-nothing"),
+("format"="json"),
+("include"="*")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp
index 598d9ff..284eb58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.001.query.sqlpp
@@ -16,23 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+use test;
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp
index 598d9ff..548e632 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/test.099.ddl.sqlpp
@@ -16,23 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp
index 598d9ff..2e3dc68 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.000.ddl.sqlpp
@@ -16,23 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
+drop dataverse test if exists;
+create dataverse test;
+use test;
-public class ParseUtil {
+drop type test if exists;
+create type test as open {
+};
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp
index 598d9ff..284eb58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.001.query.sqlpp
@@ -16,23 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+use test;
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp
index 598d9ff..548e632 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/exclude-all-files/test.099.ddl.sqlpp
@@ -16,23 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp
index 598d9ff..9abdbde 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.000.ddl.sqlpp
@@ -16,23 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
+drop dataverse test if exists;
+create dataverse test;
+use test;
-public class ParseUtil {
+drop type test if exists;
+create type test as open {
+};
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*.notRealExtension")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp
index 598d9ff..284eb58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.001.query.sqlpp
@@ -16,23 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+-- param max-warnings:json=1000
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+use test;
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp
similarity index 53%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp
index 598d9ff..548e632 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/no-files-returned/include-no-files/test.099.ddl.sqlpp
@@ -16,23 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-
-public class ParseUtil {
-
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
- private ParseUtil() {
- }
-
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
- }
-}
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm
new file mode 100644
index 0000000..c1a0ea2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/definition-points-to-nothing/result.001.adm
@@ -0,0 +1 @@
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm
new file mode 100644
index 0000000..c1a0ea2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/exclude-all-files/result.001.adm
@@ -0,0 +1 @@
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm
new file mode 100644
index 0000000..c1a0ea2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/no-files-returned/include-no-files/result.001.adm
@@ -0,0 +1 @@
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 7c4cf37..ad17afe 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -99,6 +99,24 @@
<expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null)</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset" check-warnings="true">
+ <compilation-unit name="aws/s3/no-files-returned/definition-points-to-nothing">
+ <output-dir compare="Text">aws/s3/no-files-returned/definition-points-to-nothing</output-dir>
+ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset" check-warnings="true">
+ <compilation-unit name="aws/s3/no-files-returned/exclude-all-files">
+ <output-dir compare="Text">aws/s3/no-files-returned/exclude-all-files</output-dir>
+ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset" check-warnings="true">
+ <compilation-unit name="aws/s3/no-files-returned/include-no-files">
+ <output-dir compare="Text">aws/s3/no-files-returned/include-no-files</output-dir>
+ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="include-exclude">
<test-case FilePath="external-dataset">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 46a6228..d57c433 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -202,6 +202,7 @@ public class ErrorCode {
public static final int PROPERTY_INVALID_VALUE_TYPE = 1111;
public static final int INVALID_PROPERTY_FORMAT = 1112;
public static final int INVALID_REGEX_PATTERN = 1113;
+ public static final int EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES = 1114;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java
similarity index 61%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
copy to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java
index 598d9ff..a036b7e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/NoOpWarningCollector.java
@@ -16,23 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
-import org.apache.hyracks.api.exceptions.ErrorCode;
+package org.apache.asterix.common.exceptions;
+
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
-public class ParseUtil {
+public final class NoOpWarningCollector implements IWarningCollector {
+
+ public static final IWarningCollector INSTANCE = new NoOpWarningCollector();
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
+ private NoOpWarningCollector() {
+ }
+
+ @Override
+ public void warn(Warning warning) {
+ // no-op
+ }
- private ParseUtil() {
+ @Override
+ public boolean shouldWarn() {
+ return false;
}
- public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
- String warnMessage) {
- warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
+ @Override
+ public long getTotalWarningsCount() {
+ return 0;
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java
index e2e7e3e..6d2df37 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IAdapterFactory.java
@@ -26,6 +26,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* Base interface for IGenericDatasetAdapterFactory and ITypedDatasetAdapterFactory.
@@ -71,9 +72,10 @@ public interface IAdapterFactory extends Serializable {
*
* @param serviceContext
* @param configuration
+ * @param warningCollector warning collector
* @throws AlgebricksException
* @throws HyracksDataException
*/
- void configure(IServiceContext serviceContext, Map<String, String> configuration)
- throws HyracksDataException, AlgebricksException;
+ void configure(IServiceContext serviceContext, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException;
}
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index b070552..5cf3968 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -197,6 +197,7 @@
1111 = Property \"%1$s\" expects value(s) of type %2$s
1112 = Invalid format for property \"%1$s\"
1113 = Invalid pattern %1$s
+1114 = The provided external dataset configuration returned no files from the external source
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 078316c..badf105 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -55,6 +55,7 @@ import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -100,7 +101,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
INCServiceContext serviceCtx = ctx.getJobletContext().getServiceContext();
INcApplicationContext appCtx = (INcApplicationContext) serviceCtx.getApplicationContext();
try {
- restoreExternalObjects(serviceCtx, appCtx.getLibraryManager());
+ restoreExternalObjects(serviceCtx, appCtx.getLibraryManager(), ctx.getWarningCollector());
} catch (Exception e) {
LOGGER.log(Level.INFO, "Failure restoring external objects", e);
throw HyracksDataException.create(e);
@@ -120,15 +121,15 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
}
}
- private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager)
- throws HyracksDataException, AlgebricksException {
+ private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
if (dataSourceFactory == null) {
dataSourceFactory = createExternalDataSourceFactory(configuration, libraryManager);
// create and configure parser factory
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
}
- dataSourceFactory.configure(serviceContext, configuration);
+ dataSourceFactory.configure(serviceContext, configuration, warningCollector);
}
if (dataParserFactory == null) {
// create and configure parser factory
@@ -140,8 +141,8 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
}
@Override
- public void configure(IServiceContext serviceContext, Map<String, String> configuration)
- throws HyracksDataException, AlgebricksException {
+ public void configure(IServiceContext serviceContext, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
this.configuration = configuration;
IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext();
ExternalDataUtils.validateDataSourceParameters(configuration);
@@ -149,7 +150,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
}
- dataSourceFactory.configure(serviceContext, configuration);
+ dataSourceFactory.configure(serviceContext, configuration, warningCollector);
ExternalDataUtils.validateDataParserParameters(configuration);
dataParserFactory = createDataParserFactory(configuration, appCtx.getLibraryManager());
dataParserFactory.setRecordType(recordType);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
index b51416a..b543804 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
@@ -40,6 +40,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class LookupAdapterFactory<T> implements Serializable {
@@ -78,15 +79,16 @@ public class LookupAdapterFactory<T> implements Serializable {
}
}
- public void configure(IServiceContext serviceContext, Map<String, String> configuration)
- throws HyracksDataException, AlgebricksException {
+ public void configure(IServiceContext serviceContext, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
this.configuration = configuration;
IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext();
- readerFactory = LookupReaderFactoryProvider.getLookupReaderFactory(serviceContext, configuration);
+ readerFactory =
+ LookupReaderFactoryProvider.getLookupReaderFactory(serviceContext, configuration, warningCollector);
dataParserFactory = (IRecordDataParserFactory<T>) ParserFactoryProvider
.getDataParserFactory(appCtx.getLibraryManager(), configuration);
dataParserFactory.setRecordType(recordType);
- readerFactory.configure(serviceContext, configuration);
+ readerFactory.configure(serviceContext, configuration, warningCollector);
dataParserFactory.configure(configuration);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
index efa12ef..17cbff4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
@@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public interface IExternalDataSourceFactory extends Serializable {
@@ -63,7 +64,7 @@ public interface IExternalDataSourceFactory extends Serializable {
* @param configuration
* @throws AsterixException
*/
- void configure(IServiceContext ctx, Map<String, String> configuration)
+ void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException, HyracksDataException;
/**
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index 7702dde..dba0f27 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -51,6 +51,7 @@ import org.apache.hyracks.api.application.ICCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.hdfs.dataflow.ConfFactory;
import org.apache.hyracks.hdfs.dataflow.InputSplitsFactory;
import org.apache.hyracks.hdfs.scheduler.Scheduler;
@@ -80,7 +81,8 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IInd
private static final List<String> recordReaderNames = Collections.unmodifiableList(Arrays.asList("hdfs"));
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AsterixException {
try {
this.serviceCtx = serviceCtx;
this.configuration = configuration;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 6ef5a23..67a7c93 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -35,6 +35,7 @@ import java.util.regex.PatternSyntaxException;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.WarningUtil;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -43,6 +44,8 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.util.CleanupUtils;
import software.amazon.awssdk.core.exception.SdkException;
@@ -80,7 +83,8 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
}
@Override
- public void configure(IServiceContext ctx, Map<String, String> configuration) throws AlgebricksException {
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
+ throws AlgebricksException {
this.configuration = configuration;
ICcApplicationContext ccApplicationContext = (ICcApplicationContext) ctx.getApplicationContext();
@@ -126,10 +130,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
// Get all objects in a bucket and extract the paths to files
ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container);
- String path = configuration.get(AwsS3.DEFINITION_FIELD_NAME);
- if (path != null) {
- listObjectsBuilder.prefix(path + (!path.isEmpty() && !path.endsWith("/") ? "/" : ""));
- }
+ ExternalDataUtils.AwsS3.setPrefix(configuration, listObjectsBuilder);
ListObjectsV2Response listObjectsResponse;
boolean done = false;
@@ -163,6 +164,12 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
}
}
+ // Warn if no files are returned
+ if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
+ Warning warning = WarningUtil.forAsterix(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ warningCollector.warn(warning);
+ }
+
// Partition constraints
partitionConstraint = ccApplicationContext.getClusterStateManager().getClusterLocations();
int partitionsCount = partitionConstraint.getLocations().length;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
index 6484d4e..1dd0c8b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3ReaderFactory.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class AwsS3ReaderFactory extends StreamRecordReaderFactory {
@@ -58,13 +59,13 @@ public class AwsS3ReaderFactory extends StreamRecordReaderFactory {
}
@Override
- public void configure(IServiceContext ctx, Map<String, String> configuration)
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException, HyracksDataException {
this.configuration = configuration;
// Stream factory
streamFactory = new AwsS3InputStreamFactory();
- streamFactory.configure(ctx, configuration);
+ streamFactory.configure(ctx, configuration, warningCollector);
// record reader
recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
index 98f78cc..8b258e4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
@@ -34,6 +34,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.hdfs.dataflow.ConfFactory;
public class HDFSLookupReaderFactory<T> implements ILookupReaderFactory<T> {
@@ -60,7 +61,8 @@ public class HDFSLookupReaderFactory<T> implements ILookupReaderFactory<T> {
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AsterixException {
this.serviceCtx = serviceCtx;
this.configuration = configuration;
JobConf conf = HDFSUtils.configureHDFSJobConf(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java
index d14071f..5954d74 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/http/HttpServerRecordReaderFactory.java
@@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.http.server.HttpServerConfigBuilder;
public class HttpServerRecordReaderFactory implements IRecordReaderFactory<char[]> {
@@ -86,7 +87,8 @@ public class HttpServerRecordReaderFactory implements IRecordReaderFactory<char[
}
@Override
- public void configure(IServiceContext ctx, Map<String, String> configuration) throws AlgebricksException {
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
+ throws AlgebricksException {
this.configurations = configuration;
// necessary configs
addrValue = getConfigurationValue(KEY_CONFIGURATION_ADDRESSES, true);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
index 2c07c35..b52636a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
@@ -35,6 +35,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import com.rometools.rome.feed.synd.SyndEntry;
@@ -60,7 +61,8 @@ public class RSSRecordReaderFactory implements IRecordReaderFactory<SyndEntry> {
}
@Override
- public void configure(IServiceContext serviceContext, Map<String, String> configuration) {
+ public void configure(IServiceContext serviceContext, Map<String, String> configuration,
+ IWarningCollector warningCollector) {
this.serviceContext = serviceContext;
String url = configuration.get(ExternalDataConstants.KEY_RSS_URL);
if (url == null) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
index 9bb50f6..c0a1b38 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class StreamRecordReaderFactory implements IRecordReaderFactory<char[]> {
@@ -81,11 +82,11 @@ public class StreamRecordReaderFactory implements IRecordReaderFactory<char[]> {
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration)
- throws HyracksDataException, AlgebricksException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
this.configuration = configuration;
configureInputStreamFactory(configuration);
- streamFactory.configure(serviceCtx, configuration);
+ streamFactory.configure(serviceCtx, configuration, warningCollector);
recordReaderClazz = StreamRecordReaderProvider.getRecordReaderClazz(configuration);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
index 43a3816..0fd0b95 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
@@ -38,6 +38,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -78,7 +79,8 @@ public class TwitterRecordReaderFactory implements IRecordReaderFactory<char[]>
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AsterixException {
try {
Class.forName("twitter4j.Twitter");
} catch (ClassNotFoundException e) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
index 6d91067..58ef2a4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java
@@ -43,6 +43,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.UnmanagedFileSplit;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -90,7 +91,8 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory {
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AsterixException {
this.configuration = configuration;
String[] splits = configuration.get(ExternalDataConstants.KEY_PATH).split(",");
if (inputFileSplits == null) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
index bfc2cb7..5d2b2a6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketClientInputStreamFactory.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class SocketClientInputStreamFactory implements IInputStreamFactory {
@@ -55,7 +56,8 @@ public class SocketClientInputStreamFactory implements IInputStreamFactory {
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws AsterixException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws AsterixException {
try {
this.serviceCtx = serviceCtx;
this.sockets = new ArrayList<>();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
index ac3ac42..1bd08d9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
@@ -36,6 +36,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class SocketServerInputStreamFactory implements IInputStreamFactory {
@@ -43,7 +44,8 @@ public class SocketServerInputStreamFactory implements IInputStreamFactory {
private List<Pair<String, Integer>> sockets;
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) throws CompilationException {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) throws CompilationException {
try {
sockets = FeedUtils.extractHostsPorts(configuration.get(ExternalDataConstants.KEY_MODE), serviceCtx,
configuration.get(ExternalDataConstants.KEY_SOCKETS));
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
index c7b8633..2b0bb55 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
@@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* Factory class for creating @see{TwitterFirehoseFeedAdapter}. The adapter
@@ -84,7 +85,8 @@ public class TwitterFirehoseStreamFactory implements IInputStreamFactory {
}
@Override
- public void configure(IServiceContext serviceCtx, Map<String, String> configuration) {
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector) {
this.serviceCtx = serviceCtx;
this.configuration = configuration;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index d63e8a8..7c709fb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -110,7 +110,8 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
try {
adapterFactory = (ITypedAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
adapterFactory.setOutputType(adapterOutputType);
- adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration);
+ adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration,
+ ctx.getWarningCollector());
} catch (Exception e) {
throw HyracksDataException.create(e);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
index 27ac10e..96f06f4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
@@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* This class represents the entry point to all things adapters
@@ -44,8 +45,8 @@ public class AdapterFactoryProvider {
// get adapter factory. this method has the side effect of modifying the configuration as necessary
public static ITypedAdapterFactory getAdapterFactory(IServiceContext serviceCtx, String adapterName,
- Map<String, String> configuration, ARecordType itemType, ARecordType metaType)
- throws HyracksDataException, AlgebricksException {
+ Map<String, String> configuration, ARecordType itemType, ARecordType metaType,
+ IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
ExternalDataUtils.defaultConfiguration(configuration);
ExternalDataUtils.prepare(adapterName, configuration);
ICcApplicationContext context = (ICcApplicationContext) serviceCtx.getApplicationContext();
@@ -53,32 +54,32 @@ public class AdapterFactoryProvider {
(ITypedAdapterFactory) context.getAdapterFactoryService().createAdapterFactory();
adapterFactory.setOutputType(itemType);
adapterFactory.setMetaType(metaType);
- adapterFactory.configure(serviceCtx, configuration);
+ adapterFactory.configure(serviceCtx, configuration, warningCollector);
return adapterFactory;
}
// get indexing adapter factory. this method has the side effect of modifying the configuration as necessary
public static IIndexingAdapterFactory getIndexingAdapterFactory(IServiceContext serviceCtx, String adapterName,
Map<String, String> configuration, ARecordType itemType, List<ExternalFile> snapshot, boolean indexingOp,
- ARecordType metaType) throws HyracksDataException, AlgebricksException {
+ ARecordType metaType, IWarningCollector warningCollector) throws HyracksDataException, AlgebricksException {
ExternalDataUtils.defaultConfiguration(configuration);
ExternalDataUtils.prepare(adapterName, configuration);
GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
adapterFactory.setOutputType(itemType);
adapterFactory.setMetaType(metaType);
adapterFactory.setSnapshot(snapshot, indexingOp);
- adapterFactory.configure(serviceCtx, configuration);
+ adapterFactory.configure(serviceCtx, configuration, warningCollector);
return adapterFactory;
}
// Lookup Adapters
public static LookupAdapterFactory<?> getLookupAdapterFactory(IServiceContext serviceCtx,
Map<String, String> configuration, ARecordType recordType, int[] ridFields, boolean retainInput,
- boolean retainMissing, IMissingWriterFactory missingWriterFactory)
+ boolean retainMissing, IMissingWriterFactory missingWriterFactory, IWarningCollector warningCollector)
throws HyracksDataException, AlgebricksException {
LookupAdapterFactory<?> adapterFactory =
new LookupAdapterFactory<>(recordType, ridFields, retainInput, retainMissing, missingWriterFactory);
- adapterFactory.configure(serviceCtx, configuration);
+ adapterFactory.configure(serviceCtx, configuration, warningCollector);
return adapterFactory;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java
index 69f619f..db80c2a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/LookupReaderFactoryProvider.java
@@ -26,18 +26,19 @@ import org.apache.asterix.external.input.record.reader.hdfs.HDFSLookupReaderFact
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.HDFSUtils;
import org.apache.hyracks.api.application.IServiceContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class LookupReaderFactoryProvider {
@SuppressWarnings("rawtypes")
public static ILookupReaderFactory getLookupReaderFactory(IServiceContext serviceCtx,
- Map<String, String> configuration) throws AsterixException {
+ Map<String, String> configuration, IWarningCollector warningCollector) throws AsterixException {
String inputFormat = HDFSUtils.getInputFormatClassName(configuration);
if (inputFormat.equals(ExternalDataConstants.CLASS_NAME_TEXT_INPUT_FORMAT)
|| inputFormat.equals(ExternalDataConstants.CLASS_NAME_SEQUENCE_INPUT_FORMAT)
|| inputFormat.equals(ExternalDataConstants.CLASS_NAME_RC_INPUT_FORMAT)) {
HDFSLookupReaderFactory<Object> readerFactory = new HDFSLookupReaderFactory<>();
- readerFactory.configure(serviceCtx, configuration);
+ readerFactory.configure(serviceCtx, configuration, warningCollector);
return readerFactory;
} else {
throw new AsterixException("Unrecognized external format");
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8496761..1300ac3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -37,6 +37,7 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.common.exceptions.WarningUtil;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
@@ -48,7 +49,9 @@ import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.runtime.evaluators.common.NumberUtils;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.dataflow.common.data.parsers.BooleanParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -462,13 +465,13 @@ public class ExternalDataUtils {
*
* @param configuration properties
*/
- public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
- throws CompilationException {
+ public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector) throws CompilationException {
String type = configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE);
switch (type) {
case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
- ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc);
+ ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc, collector);
break;
default:
// Nothing needs to be done
@@ -622,14 +625,27 @@ public class ExternalDataUtils {
}
/**
+ * Sets the prefix for the list objects builder if it is available
+ *
+ * @param configuration configuration
+ * @param builder builder
+ */
+ public static void setPrefix(Map<String, String> configuration, ListObjectsV2Request.Builder builder) {
+ String definition = configuration.get(ExternalDataConstants.AwsS3.DEFINITION_FIELD_NAME);
+ if (definition != null) {
+ builder.prefix(definition + (!definition.isEmpty() && !definition.endsWith("/") ? "/" : ""));
+ }
+ }
+
+ /**
* Validate external dataset properties
*
* @param configuration properties
*
* @throws CompilationException Compilation exception
*/
- public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc)
- throws CompilationException {
+ public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
+ IWarningCollector collector) throws CompilationException {
// check if the format property is present
if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
@@ -643,8 +659,17 @@ public class ExternalDataUtils {
try {
String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
s3Client = buildAwsS3Client(configuration);
+ ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder();
+ setPrefix(configuration, listObjectsBuilder);
+
ListObjectsV2Response response =
- s3Client.listObjectsV2(ListObjectsV2Request.builder().bucket(container).maxKeys(1).build());
+ s3Client.listObjectsV2(listObjectsBuilder.bucket(container).maxKeys(1).build());
+
+ if (response.contents().isEmpty() && collector.shouldWarn()) {
+ Warning warning =
+ WarningUtil.forAsterix(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+ collector.warn(warning);
+ }
// Returns 200 only in case the bucket exists, however, otherwise, throws an exception. However, to
// ensure coverage, check if the result is successful as well and not only catch exceptions
@@ -661,7 +686,6 @@ public class ExternalDataUtils {
}
/**
- * TODO(Hussain)
* @param configuration
* @throws CompilationException
*/
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
index 598d9ff..7afc592 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ParseUtil.java
@@ -20,19 +20,16 @@ package org.apache.asterix.external.util;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
public class ParseUtil {
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
-
private ParseUtil() {
}
public static void warn(IWarningCollector warningCollector, String dataSourceName, long lineNum, int fieldNum,
String warnMessage) {
warningCollector.warn(
- Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
+ Warning.forHyracks(null, ErrorCode.PARSING_ERROR, dataSourceName, lineNum, fieldNum, warnMessage));
}
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
index 715ad02..c65e00d 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
@@ -32,6 +32,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class RecordWithPKTestReaderFactory implements IRecordReaderFactory<RecordWithPK<char[]>> {
@@ -48,7 +49,8 @@ public class RecordWithPKTestReaderFactory implements IRecordReaderFactory<Recor
}
@Override
- public void configure(IServiceContext serviceCtx, final Map<String, String> configuration) {
+ public void configure(IServiceContext serviceCtx, final Map<String, String> configuration,
+ IWarningCollector warningCollector) {
this.serviceCtx = serviceCtx;
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
index c09b9eb..49ca23e 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
@@ -29,6 +29,7 @@ import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import com.couchbase.client.core.message.dcp.DCPRequest;
@@ -58,7 +59,8 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest> {
}
@Override
- public void configure(IServiceContext serviceCtx, final Map<String, String> configuration) {
+ public void configure(IServiceContext serviceCtx, final Map<String, String> configuration,
+ IWarningCollector warningCollector) {
this.serviceCtx = serviceCtx;
if (configuration.containsKey("num-of-records")) {
numOfRecords = Integer.parseInt(configuration.get("num-of-records"));
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index a947c7e..0bd601b 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -37,6 +37,7 @@ import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.std.file.ITupleParser;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -113,7 +114,8 @@ public class TestTypedAdapterFactory implements ITypedAdapterFactory {
}
@Override
- public void configure(IServiceContext serviceContext, Map<String, String> configuration) {
+ public void configure(IServiceContext serviceContext, Map<String, String> configuration,
+ IWarningCollector warningCollector) {
this.serviceContext = serviceContext;
this.configuration = configuration;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 07bbc57..71d389e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -111,8 +111,9 @@ public class DatasetDataSource extends DataSource {
externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
- ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
- edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
+ ITypedAdapterFactory adapterFactory =
+ metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(),
+ edd.getProperties(), (ARecordType) itemType, null, context.getWarningCollector());
return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
case INTERNAL:
DataSourceId id = getId();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java
index 1e0ddbf..5503cb0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java
@@ -30,6 +30,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class FunctionDataSourceFactory implements IRecordReaderFactory<char[]> {
@@ -51,7 +52,7 @@ public class FunctionDataSourceFactory implements IRecordReaderFactory<char[]> {
}
@Override
- public void configure(IServiceContext ctx, Map<String, String> configuration)
+ public void configure(IServiceContext ctx, Map<String, String> configuration, IWarningCollector warningCollector)
throws AlgebricksException, HyracksDataException {
// No Op
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index c2983af..e25ea22 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -138,7 +138,7 @@ public class LoadableDataSource extends DataSource {
LoadableDataSource alds = (LoadableDataSource) dataSource;
ARecordType itemType = (ARecordType) alds.getLoadedType();
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(),
- alds.getAdapter(), alds.getAdapterProperties(), itemType, null);
+ alds.getAdapter(), alds.getAdapterProperties(), itemType, null, context.getWarningCollector());
RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
return metadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 6b6cc78..c5109b7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -130,6 +130,7 @@ import org.apache.hyracks.api.dataflow.value.IResultSerializerFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.result.IResultMetadata;
@@ -781,11 +782,13 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
}
protected ITypedAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
- Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException {
+ Map<String, String> configuration, ARecordType itemType, ARecordType metaType,
+ IWarningCollector warningCollector) throws AlgebricksException {
try {
configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName());
- ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(
- getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType);
+ ITypedAdapterFactory adapterFactory =
+ AdapterFactoryProvider.getAdapterFactory(getApplicationContext().getServiceContext(), adapterName,
+ configuration, itemType, metaType, warningCollector);
// check to see if dataset is indexed
Index filesIndex =
@@ -858,7 +861,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getLookupAdapterFactory(
getApplicationContext().getServiceContext(), datasetDetails.getProperties(), itemType, ridIndexes,
- retainInput, retainMissing, context.getMissingWriterFactory());
+ retainInput, retainMissing, context.getMissingWriterFactory(), context.getWarningCollector());
String fileIndexName = IndexingConstants.getFilesIndexName(dataset.getDatasetName());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
metadataProvider.getSplitProviderAndConstraints(dataset, fileIndexName);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 071fd1c..792501b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -28,6 +28,7 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.MetadataException;
+import org.apache.asterix.common.exceptions.NoOpWarningCollector;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType;
import org.apache.asterix.external.api.ITypedAdapterFactory;
@@ -52,6 +53,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* A utility class for providing helper functions for feeds TODO: Refactor this
@@ -98,8 +100,8 @@ public class FeedMetadataUtil {
return feedPolicy;
}
- public static void validateFeed(Feed feed, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx)
- throws AlgebricksException {
+ public static void validateFeed(Feed feed, MetadataTransactionContext mdTxnCtx, ICcApplicationContext appCtx,
+ IWarningCollector warningCollector) throws AlgebricksException {
try {
Map<String, String> configuration = feed.getConfiguration();
ARecordType adapterOutputType = getOutputType(feed, configuration.get(ExternalDataConstants.KEY_TYPE_NAME));
@@ -141,7 +143,7 @@ public class FeedMetadataUtil {
}
adapterFactory.setOutputType(adapterOutputType);
adapterFactory.setMetaType(metaType);
- adapterFactory.configure(appCtx.getServiceContext(), configuration);
+ adapterFactory.configure(appCtx.getServiceContext(), configuration, warningCollector);
if (metaType == null && configuration.containsKey(ExternalDataConstants.KEY_META_TYPE_NAME)) {
metaType = getOutputType(feed, configuration.get(ExternalDataConstants.KEY_META_TYPE_NAME));
if (metaType == null) {
@@ -210,10 +212,10 @@ public class FeedMetadataUtil {
}
adapterFactory.setOutputType(adapterOutputType);
adapterFactory.setMetaType(metaType);
- adapterFactory.configure(appCtx.getServiceContext(), configuration);
+ adapterFactory.configure(appCtx.getServiceContext(), configuration, NoOpWarningCollector.INSTANCE);
} else {
adapterFactory = AdapterFactoryProvider.getAdapterFactory(appCtx.getServiceContext(), adapterName,
- configuration, adapterOutputType, metaType);
+ configuration, adapterOutputType, metaType, NoOpWarningCollector.INSTANCE);
adapterType = IDataSourceAdapter.AdapterType.INTERNAL;
}
if (metaType == null) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java
index c1d8f42..9847fe4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java
@@ -30,6 +30,7 @@ import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.config.DatasetConfig.TransactionState;
import org.apache.asterix.common.context.IStorageComponentProvider;
+import org.apache.asterix.common.exceptions.NoOpWarningCollector;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
@@ -256,7 +257,7 @@ public class ExternalIndexingOperations {
Map<String, String> configuration = externalDatasetDetails.getProperties();
ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getIndexingAdapterFactory(
metadataProvider.getApplicationContext().getServiceContext(), externalDatasetDetails.getAdapter(),
- configuration, (ARecordType) itemType, files, true, null);
+ configuration, (ARecordType) itemType, files, true, null, NoOpWarningCollector.INSTANCE);
ExternalScanOperatorDescriptor scanOp =
new ExternalScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory);
scanOp.setSourceLocation(sourceLoc);
diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index ab68701..0fd107f 100644
--- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -105,7 +105,7 @@ public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuil
normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer,
oc, expressionEvalSizeComputer, partialAggregationTypeComputer,
predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(),
- clusterLocations, maxWarnings);
+ clusterLocations, warningCollector, maxWarnings);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, jobEventListenerFactory);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
index ae333e2..4bc5689 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
@@ -45,6 +45,7 @@ import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider;
import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public class JobGenContext {
private final IOperatorSchema outerFlowSchema;
@@ -70,6 +71,7 @@ public class JobGenContext {
private AlgebricksAbsolutePartitionConstraint clusterLocations;
private int varCounter;
private final ITypingContext typingContext;
+ private final IWarningCollector warningCollector;
private final long maxWarnings;
public JobGenContext(IOperatorSchema outerFlowSchema, IMetadataProvider<?, ?> metadataProvider, Object appContext,
@@ -85,7 +87,8 @@ public class JobGenContext {
ITypingContext typingContext, IExpressionEvalSizeComputer expressionEvalSizeComputer,
IPartialAggregationTypeComputer partialAggregationTypeComputer,
IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize,
- AlgebricksAbsolutePartitionConstraint clusterLocations, long maxWarnings) {
+ AlgebricksAbsolutePartitionConstraint clusterLocations, IWarningCollector warningCollector,
+ long maxWarnings) {
this.outerFlowSchema = outerFlowSchema;
this.metadataProvider = metadataProvider;
this.appContext = appContext;
@@ -108,6 +111,7 @@ public class JobGenContext {
this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
this.frameSize = frameSize;
this.varCounter = 0;
+ this.warningCollector = warningCollector;
this.maxWarnings = maxWarnings;
}
@@ -209,6 +213,10 @@ public class JobGenContext {
return typingContext.getOutputTypeEnvironment(op);
}
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
+ }
+
public long getMaxWarnings() {
return maxWarnings;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java
index 3accc38..2a8636a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/SourceLocation.java
@@ -63,12 +63,20 @@ public final class SourceLocation implements Serializable {
return column;
}
- public void writeFields(DataOutput output) throws IOException {
- output.writeInt(line);
- output.writeInt(column);
+ public static void writeFields(SourceLocation sourceLocation, DataOutput output) throws IOException {
+ if (sourceLocation == null) {
+ output.writeInt(-1);
+ } else {
+ output.writeInt(sourceLocation.getLine());
+ output.writeInt(sourceLocation.getColumn());
+ }
}
public static SourceLocation create(DataInput dataInput) throws IOException {
- return new SourceLocation(dataInput.readInt(), dataInput.readInt());
+ int row = dataInput.readInt();
+ if (row == -1) {
+ return null;
+ }
+ return new SourceLocation(row, dataInput.readInt());
}
-}
\ No newline at end of file
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
index 78fe3d8..3face66 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
@@ -42,7 +42,6 @@ public class Warning implements Serializable {
}
public static Warning of(String component, SourceLocation srcLocation, int code, String message) {
- Objects.requireNonNull(srcLocation, "warnings must have source location");
return new Warning(component, srcLocation, code, message);
}
@@ -88,7 +87,7 @@ public class Warning implements Serializable {
output.writeUTF(component);
output.writeInt(code);
output.writeUTF(message);
- srcLocation.writeFields(output);
+ SourceLocation.writeFields(srcLocation, output);
}
public static Warning create(DataInput input) throws IOException {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
index 48cfca6..e401efc 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FieldCursorForDelimitedDataParser.java
@@ -25,7 +25,6 @@ import java.util.function.Supplier;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
public class FieldCursorForDelimitedDataParser {
@@ -45,7 +44,6 @@ public class FieldCursorForDelimitedDataParser {
END
}
- private static final SourceLocation SRC_LOC = new SourceLocation(-1, -1);
private static final String CLOSING_Q = "missing a closing quote";
private static final String OPENING_Q = "a quote should be in the beginning";
private static final String DELIMITER_AFTER_Q = "a quote enclosing a field needs to be followed by the delimiter";
@@ -450,7 +448,7 @@ public class FieldCursorForDelimitedDataParser {
}
private void warn(String message) {
- warnings.warn(Warning.forHyracks(SRC_LOC, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount,
+ warnings.warn(Warning.forHyracks(null, ErrorCode.PARSING_ERROR, dataSourceName.get(), lineCount, fieldCount,
message));
}
}