You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2019/05/07 16:58:34 UTC
[asterixdb] branch master updated: [NO ISSUE][CONF] Add compilation
provider to NC application context
This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1c079ba [NO ISSUE][CONF] Add compilation provider to NC application context
1c079ba is described below
commit 1c079ba5a36d8a67ee65b5b44f2ccfb452c30f77
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon May 6 10:09:42 2019 -0700
[NO ISSUE][CONF] Add compilation provider to NC application context
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Create NCExtensionManager in NCApplication instead of
NCAppRuntimeContext (aligned with how CCApplication creates
CCExtensionManager)
- Pass extension manager to application context via constructor
instead of setter method (for both CC and NC contexts)
- Move getExtensionManager() from ICcApplicationContext
to IApplicationContext, so it's available in NC application context
- Add compilation provider to NC application context
(NCExtensionManager.getCompilationProvider())
- Add compilation provider to NCQueryServiceServlet constructor
Change-Id: Ibb4dc6d8dbc00c8a701fddb1a3205d1070078cb4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3378
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../api/http/server/NCQueryServiceServlet.java | 5 ++--
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 21 ++++------------
.../apache/asterix/app/nc/NCExtensionManager.java | 29 ++++++++++++++++++++++
.../asterix/hyracks/bootstrap/CCApplication.java | 13 ++++------
.../asterix/hyracks/bootstrap/NCApplication.java | 13 +++++++---
.../asterix/common/api/IApplicationContext.java | 7 ++++++
.../common/dataflow/ICcApplicationContext.java | 14 -----------
.../runtime/utils/CcApplicationContext.java | 9 +++----
8 files changed, 61 insertions(+), 50 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 5a6bdd7..8923ae8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -39,6 +39,7 @@ import org.apache.asterix.common.exceptions.ExceptionUtils;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.messaging.api.INCMessageBroker;
import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.translator.IStatementExecutor;
import org.apache.asterix.translator.ResultProperties;
@@ -63,9 +64,9 @@ import io.netty.handler.codec.http.HttpResponseStatus;
public class NCQueryServiceServlet extends QueryServiceServlet {
public NCQueryServiceServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx,
- ILangExtension.Language queryLanguage,
+ ILangExtension.Language queryLanguage, ILangCompilationProvider compilationProvider,
Function<IServletRequest, Map<String, String>> optionalParamProvider) {
- super(ctx, paths, appCtx, queryLanguage, null, null, null, optionalParamProvider);
+ super(ctx, paths, appCtx, queryLanguage, compilationProvider, null, null, optionalParamProvider);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 182f982..e11302b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -21,9 +21,7 @@ package org.apache.asterix.app.nc;
import java.io.IOException;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -42,21 +40,18 @@ import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IReceptionistFactory;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.ActiveProperties;
-import org.apache.asterix.common.config.AsterixExtension;
import org.apache.asterix.common.config.BuildProperties;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.MessagingProperties;
import org.apache.asterix.common.config.MetadataProperties;
import org.apache.asterix.common.config.NodeProperties;
-import org.apache.asterix.common.config.PropertiesAccessor;
import org.apache.asterix.common.config.ReplicationProperties;
import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.common.config.TransactionProperties;
import org.apache.asterix.common.context.DatasetLifecycleManager;
import org.apache.asterix.common.context.DatasetMemoryManager;
import org.apache.asterix.common.context.IStorageComponentProvider;
-import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.replication.IReplicationChannel;
import org.apache.asterix.common.replication.IReplicationManager;
@@ -157,10 +152,8 @@ public class NCAppRuntimeContext implements INcApplicationContext {
private ICacheManager cacheManager;
private IConfigValidator configValidator;
- public NCAppRuntimeContext(INCServiceContext ncServiceContext, List<AsterixExtension> extensions,
- IPropertiesFactory propertiesFactory) throws AsterixException, InstantiationException,
- IllegalAccessException, ClassNotFoundException, IOException {
- List<AsterixExtension> allExtensions = new ArrayList<>();
+ public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager,
+ IPropertiesFactory propertiesFactory) {
this.ncServiceContext = ncServiceContext;
compilerProperties = propertiesFactory.newCompilerProperties();
externalProperties = propertiesFactory.newExternalProperties();
@@ -173,12 +166,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
messagingProperties = propertiesFactory.newMessagingProperties();
nodeProperties = propertiesFactory.newNodeProperties();
libraryManager = new ExternalLibraryManager();
- if (extensions != null) {
- allExtensions.addAll(extensions);
- }
- PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ncServiceContext.getAppConfig());
- allExtensions.addAll(propertiesAccessor.getExtensions());
- ncExtensionManager = new NCExtensionManager(allExtensions);
+ ncExtensionManager = extensionManager;
componentProvider = new StorageComponentProvider();
resourceIdFactory = new GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
persistedResourceRegistry = ncServiceContext.getPersistedResourceRegistry();
@@ -484,7 +472,8 @@ public class NCAppRuntimeContext implements INcApplicationContext {
.setMetadataNode(metadataNodeStub);
}
- public NCExtensionManager getNcExtensionManager() {
+ @Override
+ public NCExtensionManager getExtensionManager() {
return ncExtensionManager;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
index 6ea97d3..a1405a6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
@@ -22,13 +22,19 @@ import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.algebra.base.ILangExtension;
+import org.apache.asterix.common.api.ExtensionId;
import org.apache.asterix.common.api.IExtension;
import org.apache.asterix.common.config.AsterixExtension;
import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.metadata.api.IMetadataExtension;
import org.apache.asterix.metadata.api.INCExtensionManager;
import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider;
import org.apache.asterix.utils.ExtensionUtil;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -38,6 +44,8 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
*/
public class NCExtensionManager implements INCExtensionManager {
+ private final ILangCompilationProvider aqlCompilationProvider;
+ private final ILangCompilationProvider sqlppCompilationProvider;
private final MetadataTupleTranslatorProvider tupleTranslatorProvider;
private final List<IMetadataExtension> mdExtensions;
@@ -57,6 +65,8 @@ public class NCExtensionManager implements INCExtensionManager {
*/
public NCExtensionManager(List<AsterixExtension> list)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
+ Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
+ Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
IMetadataExtension tupleTranslatorProviderExtension = null;
mdExtensions = new ArrayList<>();
if (list != null) {
@@ -64,6 +74,12 @@ public class NCExtensionManager implements INCExtensionManager {
IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
extension.configure(extensionConf.getArgs());
switch (extension.getExtensionKind()) {
+ case LANG:
+ ILangExtension le = (ILangExtension) extension;
+ aqlcp = ExtensionUtil.extendLangCompilationProvider(ILangExtension.Language.AQL, aqlcp, le);
+ sqlppcp =
+ ExtensionUtil.extendLangCompilationProvider(ILangExtension.Language.SQLPP, sqlppcp, le);
+ break;
case METADATA:
IMetadataExtension mde = (IMetadataExtension) extension;
mdExtensions.add(mde);
@@ -75,10 +91,23 @@ public class NCExtensionManager implements INCExtensionManager {
}
}
}
+ this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second;
+ this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second;
this.tupleTranslatorProvider = tupleTranslatorProviderExtension == null ? new MetadataTupleTranslatorProvider()
: tupleTranslatorProviderExtension.getMetadataTupleTranslatorProvider();
}
+ public ILangCompilationProvider getCompilationProvider(ILangExtension.Language lang) {
+ switch (lang) {
+ case AQL:
+ return aqlCompilationProvider;
+ case SQLPP:
+ return sqlppCompilationProvider;
+ default:
+ throw new IllegalArgumentException(String.valueOf(lang));
+ }
+ }
+
public List<IMetadataExtension> getMetadataExtensions() {
return mdExtensions;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 76e1d88..99286a2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -151,13 +151,10 @@ public class CCApplication extends BaseCCApplication {
ExternalLibraryUtils.setUpExternaLibraries(libraryManager, false);
componentProvider = new StorageComponentProvider();
- List<AsterixExtension> extensions = new ArrayList<>();
- extensions.addAll(getExtensions());
- ccExtensionManager = new CCExtensionManager(extensions);
+ ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()));
IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager();
appCtx = createApplicationContext(libraryManager, globalRecoveryManager, lifecycleCoordinator,
- () -> new Receptionist("CC"), ConfigValidator::new);
- appCtx.setExtensionManager(ccExtensionManager);
+ () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager);
final CCConfig ccConfig = controllerService.getCCConfig();
if (System.getProperty("java.rmi.server.hostname") == null) {
System.setProperty("java.rmi.server.hostname", ccConfig.getClusterPublicAddress());
@@ -184,11 +181,11 @@ public class CCApplication extends BaseCCApplication {
protected ICcApplicationContext createApplicationContext(ILibraryManager libraryManager,
IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator lifecycleCoordinator,
- IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory)
- throws AlgebricksException, IOException {
+ IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory,
+ CCExtensionManager ccExtensionManager) throws AlgebricksException, IOException {
return new CcApplicationContext(ccServiceCtx, getHcc(), libraryManager, () -> MetadataManager.INSTANCE,
globalRecoveryManager, lifecycleCoordinator, new ActiveNotificationHandler(), componentProvider,
- new MetadataLockManager(), receptionistFactory, configValidatorFactory);
+ new MetadataLockManager(), receptionistFactory, configValidatorFactory, ccExtensionManager);
}
protected IGlobalRecoveryManager createGlobalRecoveryManager() throws Exception {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 9135688..c3402f9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -19,8 +19,8 @@
package org.apache.asterix.hyracks.bootstrap;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -30,6 +30,7 @@ import org.apache.asterix.api.http.server.StorageApiServlet;
import org.apache.asterix.app.config.ConfigValidator;
import org.apache.asterix.app.io.PersistedResourceRegistry;
import org.apache.asterix.app.nc.NCAppRuntimeContext;
+import org.apache.asterix.app.nc.NCExtensionManager;
import org.apache.asterix.app.nc.RecoveryManager;
import org.apache.asterix.app.replication.message.RegistrationTasksRequestMessage;
import org.apache.asterix.common.api.AsterixThreadFactory;
@@ -38,6 +39,7 @@ import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionistFactory;
import org.apache.asterix.common.config.AsterixExtension;
+import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.MessagingProperties;
@@ -84,6 +86,7 @@ public class NCApplication extends BaseNCApplication {
private static final Logger LOGGER = LogManager.getLogger();
protected INCServiceContext ncServiceCtx;
+ protected NCExtensionManager ncExtensionManager;
private INcApplicationContext runtimeContext;
private String nodeId;
private boolean stopInitiated;
@@ -124,7 +127,9 @@ public class NCApplication extends BaseNCApplication {
(controllerService).getConfiguration().getClusterPublicAddress());
}
MetadataBuiltinFunctions.init();
- runtimeContext = new NCAppRuntimeContext(ncServiceCtx, getExtensions(), getPropertiesFactory());
+
+ ncExtensionManager = new NCExtensionManager(new ArrayList<>(getExtensions()));
+ runtimeContext = new NCAppRuntimeContext(ncServiceCtx, ncExtensionManager, getPropertiesFactory());
MetadataProperties metadataProperties = runtimeContext.getMetadataProperties();
if (!metadataProperties.getNodeNames().contains(this.ncServiceCtx.getNodeId())) {
if (LOGGER.isInfoEnabled()) {
@@ -187,8 +192,8 @@ public class NCApplication extends BaseNCApplication {
webManager.add(apiServer);
}
- protected List<AsterixExtension> getExtensions() {
- return Collections.emptyList();
+ protected List<AsterixExtension> getExtensions() throws Exception {
+ return new ExtensionProperties(PropertiesAccessor.getInstance(ncServiceCtx.getAppConfig())).getExtensions();
}
protected IPropertiesFactory getPropertiesFactory() throws IOException, AsterixException {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index a9b306f..0340898 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -83,4 +83,11 @@ public interface IApplicationContext {
* @return the configuration validator
*/
IConfigValidator getConfigValidator();
+
+ /**
+ * Returns the extension manager
+ *
+ * @return the extension manager instance
+ */
+ Object getExtensionManager();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
index e4b70f6..3389962 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
@@ -87,13 +87,6 @@ public interface ICcApplicationContext extends IApplicationContext {
IStorageComponentProvider getStorageComponentProvider();
/**
- * Returns the extension manager
- *
- * @return the extension manager instance
- */
- Object getExtensionManager();
-
- /**
* @return the metadata lock manager
*/
IMetadataLockManager getMetadataLockManager();
@@ -109,13 +102,6 @@ public interface ICcApplicationContext extends IApplicationContext {
IClusterStateManager getClusterStateManager();
/**
- * Set the extension manager
- *
- * @param extensionManager
- */
- void setExtensionManager(Object extensionManager);
-
- /**
* @return the extension properties
*/
ExtensionProperties getExtensionProperties();
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
index c72bf54..a0b10c6 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java
@@ -104,11 +104,13 @@ public class CcApplicationContext implements ICcApplicationContext {
IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator ftStrategy,
IJobLifecycleListener activeLifeCycleListener, IStorageComponentProvider storageComponentProvider,
IMetadataLockManager mdLockManager, IReceptionistFactory receptionistFactory,
- IConfigValidatorFactory configValidatorFactory) throws AlgebricksException, IOException {
+ IConfigValidatorFactory configValidatorFactory, Object extensionManager)
+ throws AlgebricksException, IOException {
this.ccServiceCtx = ccServiceCtx;
this.hcc = hcc;
this.libraryManager = libraryManager;
this.activeLifeCycleListener = activeLifeCycleListener;
+ this.extensionManager = extensionManager;
// Determine whether to use old-style asterix-configuration.xml or new-style configuration.
// QQQ strip this out eventually
PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig());
@@ -221,11 +223,6 @@ public class CcApplicationContext implements ICcApplicationContext {
}
@Override
- public void setExtensionManager(Object extensionManager) {
- this.extensionManager = extensionManager;
- }
-
- @Override
public ExtensionProperties getExtensionProperties() {
return extensionProperties;
}