You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/05/08 13:16:59 UTC
[asterixdb] 06/06: Merge commit 'bc0f7e0' from branch
'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2364a2191b82e1499ed5f6646388f0a5e586b55d
Merge: b4ea906 bc0f7e0
Author: Michael Blow <mb...@apache.org>
AuthorDate: Wed May 6 10:30:06 2020 -0400
Merge commit 'bc0f7e0' from branch 'gerrit/mad-hatter'
Change-Id: I629850e0f6ac7e1afae3fb5c6e1f1f859b2ead72
asterixdb/asterix-app/pom.xml | 33 +++
.../asterix/app/translator/QueryTranslator.java | 24 +-
.../asterix/hyracks/bootstrap/CCApplication.java | 9 +-
.../org/apache/asterix/utils/FeedOperations.java | 12 +-
.../aws/AwsS3ExternalDatasetTest.java | 215 ++++++++++++++++++
.../resources/runtimets/only_external_dataset.xml | 23 ++
.../avg_mixed/avg_mixed.3.query.sqlpp | 3 +-
.../serial_avg_double.1.ddl.sqlpp} | 19 +-
.../serial_avg_double.2.update.sqlpp} | 18 +-
.../serial_avg_double.3.query.sqlpp} | 11 +-
.../serial_avg_double_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_double_null.2.update.sqlpp} | 18 +-
.../serial_avg_double_null.3.query.sqlpp} | 11 +-
.../serial_avg_empty.1.ddl.sqlpp} | 19 +-
.../serial_avg_empty.2.update.sqlpp} | 1 -
.../serial_avg_empty.3.query.sqlpp} | 6 +
.../serial_avg_float.1.ddl.sqlpp} | 19 +-
.../serial_avg_float.2.update.sqlpp} | 18 +-
.../serial_avg_float.3.query.sqlpp} | 6 +
.../serial_avg_float_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_float_null.2.update.sqlpp} | 18 +-
.../serial_avg_float_null.3.query.sqlpp} | 6 +
.../serial_avg_int16.1.ddl.sqlpp} | 19 +-
.../serial_avg_int16.2.update.sqlpp} | 18 +-
.../serial_avg_int16.3.query.sqlpp} | 6 +
.../serial_avg_int16_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int16_null.2.update.sqlpp} | 18 +-
.../serial_avg_int16_null.3.query.sqlpp} | 6 +
.../serial_avg_int32.1.ddl.sqlpp} | 19 +-
.../serial_avg_int32.2.update.sqlpp} | 18 +-
.../serial_avg_int32.3.query.sqlpp} | 6 +
.../serial_avg_int32_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int32_null.2.update.sqlpp} | 18 +-
.../serial_avg_int32_null.3.query.sqlpp} | 6 +
.../serial_avg_int64.1.ddl.sqlpp} | 19 +-
.../serial_avg_int64.2.update.sqlpp} | 18 +-
.../serial_avg_int64.3.query.sqlpp} | 6 +
.../serial_avg_int64_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int64_null.2.update.sqlpp} | 18 +-
.../serial_avg_int64_null.3.query.sqlpp} | 6 +
.../serial_avg_int8.1.ddl.sqlpp} | 19 +-
.../serial_avg_int8.2.update.sqlpp} | 18 +-
.../serial_avg_int8.3.query.sqlpp} | 6 +
.../serial_avg_int8_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int8_null.2.update.sqlpp} | 18 +-
.../serial_avg_int8_null.3.query.sqlpp} | 6 +
.../serial_avg_mixed.1.ddl.sqlpp} | 19 +-
.../serial_avg_mixed.2.update.sqlpp} | 16 +-
.../serial_avg_mixed.3.query.sqlpp} | 18 +-
.../aggregate/avg_mixed/avg_mixed.3.query.sqlpp | 1 +
.../serial_avg_double.1.ddl.sqlpp} | 19 +-
.../serial_avg_double.2.update.sqlpp} | 18 +-
.../serial_avg_double.3.query.sqlpp} | 11 +-
.../serial_avg_double_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_double_null.2.update.sqlpp} | 18 +-
.../serial_avg_double_null.3.query.sqlpp} | 11 +-
.../serial_avg_empty/serial_avg_empty.1.ddl.sqlpp} | 19 +-
.../serial_avg_empty.2.update.sqlpp} | 1 -
.../serial_avg_empty.3.query.sqlpp} | 11 +-
.../serial_avg_float/serial_avg_float.1.ddl.sqlpp} | 19 +-
.../serial_avg_float.2.update.sqlpp} | 18 +-
.../serial_avg_float.3.query.sqlpp} | 11 +-
.../serial_avg_float_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_float_null.2.update.sqlpp} | 18 +-
.../serial_avg_float_null.3.query.sqlpp} | 11 +-
.../serial_avg_int16/serial_avg_int16.1.ddl.sqlpp} | 19 +-
.../serial_avg_int16.2.update.sqlpp} | 18 +-
.../serial_avg_int16.3.query.sqlpp} | 11 +-
.../serial_avg_int16_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int16_null.2.update.sqlpp} | 18 +-
.../serial_avg_int16_null.3.query.sqlpp} | 11 +-
.../serial_avg_int32/serial_avg_int32.1.ddl.sqlpp} | 19 +-
.../serial_avg_int32.2.update.sqlpp} | 18 +-
.../serial_avg_int32.3.query.sqlpp} | 11 +-
.../serial_avg_int32_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int32_null.2.update.sqlpp} | 18 +-
.../serial_avg_int32_null.3.query.sqlpp} | 11 +-
.../serial_avg_int64/serial_avg_int64.1.ddl.sqlpp} | 19 +-
.../serial_avg_int64.2.update.sqlpp} | 18 +-
.../serial_avg_int64.3.query.sqlpp} | 11 +-
.../serial_avg_int64_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int64_null.2.update.sqlpp} | 18 +-
.../serial_avg_int64_null.3.query.sqlpp} | 11 +-
.../serial_avg_int8/serial_avg_int8.1.ddl.sqlpp} | 19 +-
.../serial_avg_int8.2.update.sqlpp} | 18 +-
.../serial_avg_int8/serial_avg_int8.3.query.sqlpp} | 11 +-
.../serial_avg_int8_null.1.ddl.sqlpp} | 19 +-
.../serial_avg_int8_null.2.update.sqlpp} | 18 +-
.../serial_avg_int8_null.3.query.sqlpp} | 11 +-
.../serial_avg_mixed/serial_avg_mixed.1.ddl.sqlpp} | 19 +-
.../serial_avg_mixed.2.update.sqlpp} | 16 +-
.../serial_avg_mixed.3.query.sqlpp} | 17 +-
.../aws/s3/000/external_dataset.000.ddl.sqlpp} | 23 +-
.../aws/s3/000/external_dataset.001.query.sqlpp} | 6 +
.../aws/s3/000/external_dataset.002.ddl.sqlpp} | 1 +
.../aggregate-sql/avg_mixed/avg_mixed.1.adm | 2 +-
.../serial_avg_double/serial_avg_double.1.adm | 1 +
.../serial_avg_double_null.1.adm | 1 +
.../serial_avg_empty/serial_sum_empty.1.adm} | 0
.../serial_avg_float/serial_avg_float.1.adm | 1 +
.../serial_avg_float_null.1.adm | 1 +
.../serial_avg_int16/serial_avg_int16.1.adm | 1 +
.../serial_avg_int16_null.1.adm | 1 +
.../serial_avg_int32/serial_avg_int32.1.adm | 1 +
.../serial_avg_int32_null.1.adm | 1 +
.../serial_avg_int64/serial_avg_int64.1.adm | 1 +
.../serial_avg_int64_null.1.adm | 1 +
.../serial_avg_int8/serial_avg_int8.1.adm | 1 +
.../serial_avg_int8_null.1.adm | 1 +
.../serial_avg_mixed/serial_avg_mixed.1.adm | 1 +
.../results/aggregate/avg_mixed/avg_mixed.1.adm | 2 +-
.../serial_avg_double/serial_avg_double.1.adm | 1 +
.../serial_avg_double_null.1.adm | 1 +
.../serial_avg_empty/serial_sum_empty.1.adm} | 0
.../serial_avg_float/serial_avg_float.1.adm | 1 +
.../serial_avg_float_null.1.adm | 1 +
.../serial_avg_int16/serial_avg_int16.1.adm | 1 +
.../serial_avg_int16_null.1.adm | 1 +
.../serial_avg_int32/serial_avg_int32.1.adm | 1 +
.../serial_avg_int32_null.1.adm | 1 +
.../serial_avg_int64/serial_avg_int64.1.adm | 1 +
.../serial_avg_int64_null.1.adm | 1 +
.../serial_avg_int8/serial_avg_int8.1.adm | 1 +
.../serial_avg_int8_null.1.adm | 1 +
.../serial_avg_mixed/serial_avg_mixed.1.adm | 1 +
.../aws/s3/000/external_dataset.001.adm | 14 ++
.../results/feeds/feeds_01/feeds_01.1.adm | 2 +-
.../aggregate-sql/avg_mixed/avg_mixed.1.ast | 0
.../aggregate-sql/avg_mixed/avg_mixed.2.ast | 0
.../aggregate-sql/avg_mixed/avg_mixed.3.ast | 2 +-
.../runtimets/testsuite_external_dataset.xml | 28 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 150 ++++++++++++-
.../common/dataflow/ICcApplicationContext.java | 16 ++
.../asterix/common/external}/IAdapterFactory.java | 31 +--
.../common/external/IAdapterFactoryService.java} | 16 +-
.../common/external}/IDataSourceAdapter.java | 2 +-
asterixdb/asterix-external-data/pom.xml | 12 +
.../factory/AdapterFactoryService.java} | 18 +-
.../adapter/factory/GenericAdapterFactory.java | 6 +-
.../external/api/IIndexingAdapterFactory.java | 2 +-
.../asterix/external/api/ITypedAdapterFactory.java | 54 +++++
.../external/dataset/adapter/FeedAdapter.java | 2 +-
.../external/dataset/adapter/GenericAdapter.java | 2 +-
.../input/record/reader/aws/AwsS3InputStream.java | 163 ++++++++++++++
.../record/reader/aws/AwsS3InputStreamFactory.java | 250 +++++++++++++++++++++
.../record/reader/aws/AwsS3ReaderFactory.java | 90 ++++++++
.../operators/ExternalScanOperatorDescriptor.java | 8 +-
.../operators/FeedIntakeOperatorDescriptor.java | 16 +-
.../operators/FeedIntakeOperatorNodePushable.java | 4 +-
.../external/provider/AdapterFactoryProvider.java | 9 +-
.../provider/DatasourceFactoryProvider.java | 1 +
.../util/ExternalDataCompatibilityUtils.java | 7 +-
.../external/util/ExternalDataConstants.java | 23 ++
...pache.asterix.external.api.IRecordReaderFactory | 1 +
.../library/adapter/TestTypedAdapterFactory.java | 6 +-
.../asterix/lang/common/statement/DatasetDecl.java | 4 +-
.../common/util/DatasetDeclParametersUtil.java | 18 +-
.../metadata/bootstrap/MetadataBootstrap.java | 7 +-
.../metadata/declared/DatasetDataSource.java | 4 +-
.../metadata/declared/LoadableDataSource.java | 4 +-
.../metadata/declared/MetadataProvider.java | 18 +-
.../metadata/entities/DatasourceAdapter.java | 2 +-
.../DatasourceAdapterTupleTranslator.java | 2 +-
.../asterix/metadata/feeds/FeedMetadataUtil.java | 22 +-
.../metadata/utils/ExternalIndexingOperations.java | 4 +-
.../AbstractSerializableAvgAggregateFunction.java | 36 ++-
.../std/AbstractAvgAggregateFunction.java | 35 ++-
.../runtime/utils/CcApplicationContext.java | 12 +-
asterixdb/asterix-server/pom.xml | 6 +
asterixdb/pom.xml | 79 +++++++
...streams_reactive-streams-jvm_v1.0.2_COPYING.txt | 121 ++++++++++
...streams_reactive-streams-jvm_v1.0.2_LICENSE.txt | 8 +
172 files changed, 2402 insertions(+), 559 deletions(-)
diff --cc asterixdb/asterix-app/pom.xml
index e5e978a,219595b..a26a335
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@@ -819,18 -699,38 +819,51 @@@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.mindrot</groupId>
+ <artifactId>jbcrypt</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- AWS -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sdk-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>regions</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>auth</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- Mock for AWS S3 -->
+ <dependency>
+ <groupId>io.findify</groupId>
+ <artifactId>s3mock_2.12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- Needed for the s3 mock -->
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-http-core_2.12</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 067ee6e,b8a048d..eefb7f3
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -78,17 -76,11 +78,17 @@@ import org.apache.asterix.common.except
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
++import org.apache.asterix.common.external.IDataSourceAdapter;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.utils.JobUtils;
import org.apache.asterix.common.utils.JobUtils.ProgressState;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
- import org.apache.asterix.external.api.IDataSourceAdapter;
+import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable;
@@@ -168,7 -153,10 +168,9 @@@ import org.apache.asterix.metadata.util
import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.metadata.utils.MetadataConstants;
-import org.apache.asterix.metadata.utils.MetadataLockUtil;
import org.apache.asterix.metadata.utils.MetadataUtil;
+ import org.apache.asterix.object.base.AdmObjectNode;
+ import org.apache.asterix.object.base.AdmStringNode;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index bfc8154,fc912b0..ee5b957
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@@ -73,10 -69,11 +73,12 @@@ import org.apache.asterix.common.config
import org.apache.asterix.common.config.ReplicationProperties;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+ import org.apache.asterix.common.external.IAdapterFactoryService;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.utils.Servlets;
+ import org.apache.asterix.external.adapter.factory.AdapterFactoryService;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.file.StorageComponentProvider;
import org.apache.asterix.messaging.CCMessageBroker;
@@@ -154,13 -149,14 +156,13 @@@ public class CCApplication extends Base
ReplicationProperties repProp =
new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
- ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, false, ccServiceCtx.getServerCtx().getAppDir());
componentProvider = new StorageComponentProvider();
-
ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()));
IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager();
+ ILibraryManager libraryManager = new ExternalLibraryManager(ccServiceCtx.getServerCtx().getAppDir(),
+ ccServiceCtx.getPersistedResourceRegistry());
appCtx = createApplicationContext(libraryManager, globalRecoveryManager, lifecycleCoordinator,
- () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager);
+ () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager, new AdapterFactoryService());
final CCConfig ccConfig = controllerService.getCCConfig();
if (System.getProperty("java.rmi.server.hostname") == null) {
System.setProperty("java.rmi.server.hostname", ccConfig.getClusterPublicAddress());
@@@ -208,11 -184,12 +210,12 @@@
protected ICcApplicationContext createApplicationContext(ILibraryManager libraryManager,
IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator lifecycleCoordinator,
IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory,
- CCExtensionManager ccExtensionManager) throws AlgebricksException, IOException {
+ CCExtensionManager ccExtensionManager, IAdapterFactoryService adapterFactoryService)
+ throws AlgebricksException, IOException {
return new CcApplicationContext(ccServiceCtx, getHcc(), libraryManager, () -> MetadataManager.INSTANCE,
globalRecoveryManager, lifecycleCoordinator, new ActiveNotificationHandler(), componentProvider,
- new MetadataLockManager(), receptionistFactory, configValidatorFactory, ccExtensionManager,
- adapterFactoryService);
+ new MetadataLockManager(), createMetadataLockUtil(), receptionistFactory, configValidatorFactory,
- ccExtensionManager);
++ ccExtensionManager, adapterFactoryService);
}
protected IGlobalRecoveryManager createGlobalRecoveryManager() throws Exception {
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
index d764b98,5fc1bb7..ad90814
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
@@@ -26,8 -27,8 +27,9 @@@ import org.apache.asterix.common.cluste
import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.context.IStorageComponentProvider;
+ import org.apache.asterix.common.external.IAdapterFactoryService;
import org.apache.asterix.common.metadata.IMetadataBootstrap;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.IResourceIdManager;
diff --cc asterixdb/asterix-external-data/pom.xml
index 08ffb16,30e7770..f6ebdba
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@@ -445,16 -436,16 +445,28 @@@
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>pyrolite</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>serpent</artifactId>
+ </dependency>
++ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>regions</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>auth</artifactId>
+ </dependency>
</dependencies>
</project>
diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index 3b2d95f,d63e8a8..779b05f
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@@ -24,13 -24,10 +24,13 @@@ import org.apache.asterix.active.Entity
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
- import org.apache.asterix.external.api.IAdapterFactory;
+ import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
@@@ -108,15 -105,10 +108,15 @@@ public class FeedIntakeOperatorDescript
INcApplicationContext runtimeCtx =
(INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
- ClassLoader classLoader = libraryManager.getLibraryClassLoader(feedId.getDataverse(), adaptorLibraryName);
+
+ ILibrary lib = libraryManager.getLibrary(feedId.getDataverseName(), adaptorLibraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader();
if (classLoader != null) {
try {
- adapterFactory = (IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
+ adapterFactory = (ITypedAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
adapterFactory.setOutputType(adapterOutputType);
adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration);
} catch (Exception e) {
diff --cc asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index b6a3d66,45fc33a..811a494
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@@ -45,22 -44,30 +45,22 @@@ public class DatasetDecl extends Abstra
protected final DatasetType datasetType;
protected final IDatasetDetailsDecl datasetDetailsDecl;
protected final Map<String, String> hints;
- private final AdmObjectNode withObjectNode;
+ private AdmObjectNode withObjectNode;
protected final boolean ifNotExists;
- public DatasetDecl(Identifier dataverse, Identifier name, Identifier itemTypeDataverse, Identifier itemTypeName,
- Identifier metaItemTypeDataverse, Identifier metaItemTypeName, Identifier nodeGroupName,
- Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, RecordConstructor withRecord,
- boolean ifNotExists) throws CompilationException {
+ public DatasetDecl(DataverseName dataverse, Identifier name, DataverseName itemTypeDataverse,
+ Identifier itemTypeName, DataverseName metaItemTypeDataverse, Identifier metaItemTypeName,
+ Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd,
+ RecordConstructor withRecord, boolean ifNotExists) throws CompilationException {
this.dataverse = dataverse;
this.name = name;
this.itemTypeName = itemTypeName;
- if (itemTypeDataverse.getValue() == null) {
- this.itemTypeDataverse = dataverse;
- } else {
- this.itemTypeDataverse = itemTypeDataverse;
- }
+ this.itemTypeDataverse = itemTypeDataverse == null ? dataverse : itemTypeDataverse;
this.metaItemTypeName = metaItemTypeName;
- if (metaItemTypeDataverse == null || metaItemTypeDataverse.getValue() == null) {
- this.metaItemTypeDataverse = dataverse;
- } else {
- this.metaItemTypeDataverse = metaItemTypeDataverse;
- }
+ this.metaItemTypeDataverse = metaItemTypeDataverse == null ? dataverse : metaItemTypeDataverse;
this.nodegroupName = nodeGroupName;
this.hints = hints;
- this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord);
+ this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord, datasetType);
this.ifNotExists = ifNotExists;
this.datasetType = datasetType;
this.datasetDetailsDecl = idd;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index f5ca79a,07bbc57..81646e20
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@@ -23,8 -23,7 +23,8 @@@ import java.util.List
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.metadata.DataverseName;
- import org.apache.asterix.external.api.IAdapterFactory;
+ import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.entities.Dataset;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index e1ad67e,c2983af..4d4b17a
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@@ -26,8 -26,7 +26,8 @@@ import java.util.Map
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.metadata.DataverseName;
- import org.apache.asterix.external.api.IAdapterFactory;
+ import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.types.ARecordType;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index ff68e78,5bdf2a7..93e214e
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@@ -40,7 -39,7 +40,8 @@@ import org.apache.asterix.common.contex
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.exceptions.AsterixException;
+ import org.apache.asterix.common.external.IDataSourceAdapter;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.LockList;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.ITxnIdFactory;
@@@ -817,11 -775,11 +817,11 @@@ public class MetadataProvider implement
return numElementsHint / numPartitions;
}
- protected IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
+ protected ITypedAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException {
try {
- configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName());
+ configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName().getCanonicalForm());
- IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(
+ ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(
getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType);
// check to see if dataset is indexed
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
index a94adfa,9e65c08..eb37a60
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
@@@ -19,13 -19,14 +19,13 @@@
package org.apache.asterix.metadata.entitytupletranslators;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
+import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.DATASOURCE_ARECORD_FUNCTION_LIBRARY_FIELD_NAME;
+
import java.util.Calendar;
+ import org.apache.asterix.common.external.IDataSourceAdapter;
+import org.apache.asterix.common.metadata.DataverseName;
- import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 790d7dd,7ed53e4..09eaa8b
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@@ -28,15 -28,11 +28,15 @@@ import org.apache.asterix.common.except
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.MetadataException;
+ import org.apache.asterix.common.external.IDataSourceAdapter;
+ import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
+import org.apache.asterix.common.metadata.DataverseName;
- import org.apache.asterix.external.api.IAdapterFactory;
- import org.apache.asterix.external.api.IDataSourceAdapter;
- import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType;
+ import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
@@@ -133,12 -128,9 +133,12 @@@ public class FeedMetadataUtil
case EXTERNAL:
String[] anameComponents = adapterName.split("#");
String libraryName = anameComponents[0];
- ClassLoader cl =
- appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
+ ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader cl = ((JavaLibrary) lib).getClassLoader();
- adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
+ adapterFactory = (ITypedAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
break;
default:
throw new AsterixException("Unknown Adapter type " + adapterType);
@@@ -209,12 -201,9 +209,12 @@@
case EXTERNAL:
String[] anameComponents = adapterName.split("#");
String libraryName = anameComponents[0];
- ClassLoader cl =
- appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
+ ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader cl = ((JavaLibrary) lib).getClassLoader();
- adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
+ adapterFactory = (ITypedAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
break;
default:
throw new AsterixException("Unknown Adapter type " + adapterType);
diff --cc asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
index 3de4a88,3366ac1..6def41a
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
@@@ -45,9 -45,9 +45,10 @@@ import org.apache.asterix.common.config
import org.apache.asterix.common.config.TransactionProperties;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+ import org.apache.asterix.common.external.IAdapterFactoryService;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.IMetadataBootstrap;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.IResourceIdManager;
@@@ -105,9 -105,9 +107,9 @@@ public class CcApplicationContext imple
ILibraryManager libraryManager, Supplier<IMetadataBootstrap> metadataBootstrapSupplier,
IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator ftStrategy,
IJobLifecycleListener activeLifeCycleListener, IStorageComponentProvider storageComponentProvider,
- IMetadataLockManager mdLockManager, IReceptionistFactory receptionistFactory,
+ IMetadataLockManager mdLockManager, IMetadataLockUtil mdLockUtil, IReceptionistFactory receptionistFactory,
- IConfigValidatorFactory configValidatorFactory, Object extensionManager)
- throws AlgebricksException, IOException {
+ IConfigValidatorFactory configValidatorFactory, Object extensionManager,
+ IAdapterFactoryService adapterFactoryService) throws AlgebricksException, IOException {
this.ccServiceCtx = ccServiceCtx;
this.hcc = hcc;
this.libraryManager = libraryManager;
diff --cc asterixdb/asterix-server/pom.xml
index e6d7b69,7acfc04..b660379
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@@ -173,9 -173,10 +173,14 @@@
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.46.Final/NOTICE.txt</noticeUrl>
</override>
<override>
+ <gav>org.reactivestreams:reactive-streams:1.0.2</gav>
+ <noticeUrl>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/COPYING.txt</noticeUrl>
+ <url>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/LICENSE.txt</url>
+ </override>
++ <override>
+ <gav>org.mindrot:jbcrypt:0.4</gav>
+ <url>http://www.mindrot.org/files/jBCrypt/LICENSE</url>
+ </override>
</overrides>
<licenses>
<license>
diff --cc asterixdb/pom.xml
index 69ee90c,76ec3d2..7d58b73
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -1358,40 -1342,83 +1359,118 @@@
<version>0.9.12</version>
</dependency>
<dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <version>${awsjavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>regions</artifactId>
+ <version>${awsjavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>auth</artifactId>
+ <version>${awsjavasdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sdk-core</artifactId>
+ <version>${awsjavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Mock for AWS S3 -->
+ <dependency>
+ <groupId>io.findify</groupId>
+ <artifactId>s3mock_2.12</artifactId>
+ <version>0.2.5</version>
+ </dependency>
+ <!-- Needed for the s3 mock -->
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-http-core_2.12</artifactId>
+ <version>10.1.0</version>
+ </dependency>
++ <dependency>
+ <groupId>org.mindrot</groupId>
+ <artifactId>jbcrypt</artifactId>
+ <version>0.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>1.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>42.2.10</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>4.5.11</version>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>pyrolite</artifactId>
+ <version>4.30</version>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>serpent</artifactId>
+ <version>1.23</version>
+ </dependency>
</dependencies>
</dependencyManagement>