You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2016/07/08 12:50:22 UTC
[2/2] asterixdb git commit: ASTERIXDB-1494: refactor
ExternalLibraryManager and make it non-static.
ASTERIXDB-1494: refactor ExternalLibraryManager and make it non-static.
Change-Id: I5c287a35ff90c3aea639d3069d5a842e28c5e508
Reviewed-on: https://asterix-gerrit.ics.uci.edu/980
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f09607dd
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f09607dd
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f09607dd
Branch: refs/heads/master
Commit: f09607dd9328b23aa50bd274628dcae6cbbcf493
Parents: 20e085b
Author: Yingyi Bu <yi...@couchbase.com>
Authored: Thu Jul 7 22:58:57 2016 -0700
Committer: Yingyi Bu <bu...@gmail.com>
Committed: Fri Jul 8 05:49:52 2016 -0700
----------------------------------------------------------------------
.../physical/ExternalDataLookupPOperator.java | 4 +-
.../api/common/AsterixAppRuntimeContext.java | 17 +++++-
.../external/ExternalIndexingOperations.java | 9 +--
.../app/external/ExternalLibraryUtils.java | 31 ++++++----
.../asterix/aql/translator/QueryTranslator.java | 2 +-
.../bootstrap/CCApplicationEntryPoint.java | 10 +++-
.../bootstrap/NCApplicationEntryPoint.java | 8 +--
.../asterix/app/external/TestLibrarian.java | 35 +++++++++---
.../asterix/test/runtime/ExecutionTest.java | 12 ++--
.../asterix/test/runtime/ExecutionTestUtil.java | 20 ++++---
.../asterix/test/runtime/RepeatedTest.java | 4 --
.../common/api/IAsterixAppRuntimeContext.java | 3 +
.../IAsterixApplicationContextInfo.java | 9 +++
.../asterix/common/library/ILibraryManager.java | 59 ++++++++++++++++++++
.../adapter/factory/GenericAdapterFactory.java | 24 ++++----
.../adapter/factory/LookupAdapterFactory.java | 6 +-
.../asterix/external/api/IAdapterFactory.java | 5 +-
.../external/library/ExternalFunction.java | 15 ++++-
.../library/ExternalLibraryManager.java | 27 +++++----
.../operators/FeedIntakeOperatorDescriptor.java | 29 +++++-----
.../parser/factory/RSSParserFactory.java | 1 +
.../parser/factory/TweetParserFactory.java | 1 +
.../provider/AdapterFactoryProvider.java | 19 ++++---
.../provider/DatasourceFactoryProvider.java | 22 ++++----
.../provider/ParserFactoryProvider.java | 8 ++-
.../external/util/ExternalDataUtils.java | 25 ++++-----
.../adapter/TestTypedAdapterFactory.java | 11 ++--
.../factory/TestRecordWithPKParserFactory.java | 3 +-
.../metadata/declared/AqlMetadataProvider.java | 31 ++++++----
.../metadata/feeds/FeedMetadataUtil.java | 44 +++++++--------
.../asterix/om/util/AsterixAppContextInfo.java | 14 ++++-
.../cc/application/CCApplicationContext.java | 11 ++--
32 files changed, 337 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 7db93e8..0fb6be7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -124,7 +124,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
- throws AlgebricksException {
+ throws AlgebricksException {
UnnestMapOperator unnestMap = (UnnestMapOperator) op;
ILogicalExpression expr = unnestMap.getExpressionRef().getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
@@ -145,7 +145,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
}
AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
- Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = AqlMetadataProvider
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = metadataProvider
.buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex, ridIndexes, retainInput,
typeEnv, outputVars, opSchema, context, metadataProvider, retainNull);
builder.contributeHyracksOperator(unnestMap, externalLoopup.first);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 5b3e453..1545325 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -43,6 +43,7 @@ import org.apache.asterix.common.context.AsterixFileMapManager;
import org.apache.asterix.common.context.DatasetLifecycleManager;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.replication.IReplicaResourcesManager;
import org.apache.asterix.common.replication.IReplicationChannel;
@@ -52,6 +53,7 @@ import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.external.feed.management.FeedManager;
+import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataNode;
import org.apache.asterix.metadata.api.IAsterixStateProxy;
@@ -127,6 +129,8 @@ public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAst
private IReplicaResourcesManager replicaResourcesManager;
private final int metadataRmiPort;
+ private ILibraryManager libraryManager;
+
public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext, int metadataRmiPort)
throws AsterixException {
this.ncApplicationContext = ncApplicationContext;
@@ -265,6 +269,11 @@ public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAst
lccm.register((ILifeCycleComponent) datasetLifecycleManager);
lccm.register((ILifeCycleComponent) txnSubsystem.getTransactionManager());
lccm.register((ILifeCycleComponent) txnSubsystem.getLockManager());
+
+ /**
+ * Initializes the library manager.
+ */
+ libraryManager = new ExternalLibraryManager();
}
@Override
@@ -407,13 +416,18 @@ public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAst
}
@Override
+ public ILibraryManager getLibraryManager() {
+ return libraryManager;
+ }
+
+ @Override
public void initializeResourceIdFactory() throws HyracksDataException {
resourceIdFactory = new GlobalResourceIdFactoryProvider(ncApplicationContext).createResourceIdFactory();
}
@Override
public void initializeMetadata(boolean newUniverse) throws Exception {
- IAsterixStateProxy proxy = null;
+ IAsterixStateProxy proxy;
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Bootstrapping metadata");
}
@@ -446,4 +460,5 @@ public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAst
public void unexportMetadataNodeStub() throws RemoteException {
UnicastRemoteObject.unexportObject(MetadataNode.INSTANCE, false);
}
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
index d4c3b0e..6922379 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
@@ -258,12 +258,13 @@ public class ExternalIndexingOperations {
* @throws Exception
*/
private static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> getExternalDataIndexingOperator(
- JobSpecification jobSpec, IAType itemType, Dataset dataset, List<ExternalFile> files,
- RecordDescriptor indexerDesc) throws AsterixException {
+ AqlMetadataProvider metadataProvider, JobSpecification jobSpec, IAType itemType, Dataset dataset,
+ List<ExternalFile> files, RecordDescriptor indexerDesc) throws AsterixException {
ExternalDatasetDetails externalDatasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
Map<String, String> configuration = externalDatasetDetails.getProperties();
IAdapterFactory adapterFactory = AdapterFactoryProvider.getIndexingAdapterFactory(
- externalDatasetDetails.getAdapter(), configuration, (ARecordType) itemType, files, true, null);
+ metadataProvider.getLibraryManager(), externalDatasetDetails.getAdapter(), configuration,
+ (ARecordType) itemType, files, true, null);
return new Pair<>(new ExternalDataScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory),
adapterFactory.getPartitionConstraint());
}
@@ -274,7 +275,7 @@ public class ExternalIndexingOperations {
if (files == null) {
files = MetadataManager.INSTANCE.getDatasetExternalFiles(metadataProvider.getMetadataTxnContext(), dataset);
}
- return getExternalDataIndexingOperator(spec, itemType, dataset, files, indexerDesc);
+ return getExternalDataIndexingOperator(metadataProvider, spec, itemType, dataset, files, indexerDesc);
}
/**
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 04aebf5..92ef062 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -35,10 +35,10 @@ import javax.xml.bind.Unmarshaller;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.external.library.ExternalLibrary;
-import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.external.library.LibraryAdapter;
import org.apache.asterix.external.library.LibraryFunction;
import org.apache.asterix.metadata.MetadataManager;
@@ -52,10 +52,13 @@ import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
public class ExternalLibraryUtils {
- private static Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
+ private static final Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
- public static void setUpExternaLibraries(boolean isMetadataNode) throws Exception {
+ private ExternalLibraryUtils() {
+ }
+ public static void setUpExternaLibraries(ILibraryManager externalLibraryManager, boolean isMetadataNode)
+ throws Exception {
// start by un-installing removed libraries (Metadata Node only)
Map<String, List<String>> uninstalledLibs = null;
if (isMetadataNode) {
@@ -72,7 +75,7 @@ public class ExternalLibraryUtils {
String[] libraries = dataverseDir.list();
for (String library : libraries) {
// for each file (library), register library
- registerLibrary(dataverse, library, isMetadataNode, installLibDir);
+ registerLibrary(externalLibraryManager, dataverse, library);
// is metadata node?
if (isMetadataNode) {
// get library file
@@ -88,6 +91,7 @@ public class ExternalLibraryUtils {
/**
* un-install libraries.
+ *
* @return a map from dataverse -> list of uninstalled libraries.
* @throws Exception
*/
@@ -126,6 +130,7 @@ public class ExternalLibraryUtils {
* TODO Currently, external libraries only include functions and adapters. we need to extend this to include:
* 1. external data source
* 2. data parser
+ *
* @param dataverse
* @param libraryName
* @return true if the library was found and removed, false otherwise
@@ -181,9 +186,9 @@ public class ExternalLibraryUtils {
}
/**
- * Each element of a library is installed as part of a transaction. Any
- * failure in installing an element does not effect installation of other
- * libraries.
+ * Each element of a library is installed as part of a transaction. Any
+ * failure in installing an element does not effect installation of other
+ * libraries.
*/
protected static void installLibraryIfNeeded(String dataverse, final File libraryDir,
Map<String, List<String>> uninstalledLibs) throws Exception {
@@ -284,22 +289,23 @@ public class ExternalLibraryUtils {
/**
* register the library class loader with the external library manager
+ *
* @param dataverse
* @param libraryName
- * @param isMetadataNode
* @param installLibDir
* @throws Exception
*/
- protected static void registerLibrary(String dataverse, String libraryName, boolean isMetadataNode,
- File installLibDir) throws Exception {
+ protected static void registerLibrary(ILibraryManager externalLibraryManager, String dataverse, String libraryName)
+ throws Exception {
// get the class loader
ClassLoader classLoader = getLibraryClassLoader(dataverse, libraryName);
// register it with the external library manager
- ExternalLibraryManager.registerLibraryClassLoader(dataverse, libraryName, classLoader);
+ externalLibraryManager.registerLibraryClassLoader(dataverse, libraryName, classLoader);
}
/**
* Get the library from the xml file
+ *
* @param libraryXMLPath
* @return
* @throws Exception
@@ -313,6 +319,7 @@ public class ExternalLibraryUtils {
/**
* Get the class loader for the library
+ *
* @param dataverse
* @param libraryName
* @return
@@ -382,7 +389,7 @@ public class ExternalLibraryUtils {
}
/**
- * @return the directory "$(pwd)/library": This needs to be improved
+ * @return the directory "$(pwd)/library": This needs to be improved
*/
protected static File getLibraryInstallDir() {
String workingDir = System.getProperty("user.dir");
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 95afc5b..295b308 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -2003,7 +2003,7 @@ public class QueryTranslator extends AbstractLangTranslator {
default:
throw new IllegalStateException();
}
- FeedMetadataUtil.validateFeed(feed, mdTxnCtx);
+ FeedMetadataUtil.validateFeed(feed, mdTxnCtx, metadataProvider.getLibraryManager());
MetadataManager.INSTANCE.addFeed(metadataProvider.getMetadataTxnContext(), feed);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 4c42594..06ec6bc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -40,10 +40,12 @@ import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.AsterixMetadataProperties;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.utils.ServletUtil.Servlets;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.event.service.ILookupService;
+import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.messaging.CCMessageBroker;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.api.IAsterixStateProxy;
@@ -87,8 +89,10 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
appCtx.setThreadFactory(new AsterixThreadFactory(new LifeCycleComponentManager()));
GlobalRecoveryManager.INSTANCE = new GlobalRecoveryManager((HyracksConnection) getNewHyracksClientConnection());
-
- AsterixAppContextInfo.initialize(appCtx, getNewHyracksClientConnection(), GlobalRecoveryManager.INSTANCE);
+ ILibraryManager libraryManager = new ExternalLibraryManager();
+ ExternalLibraryUtils.setUpExternaLibraries(libraryManager, false);
+ AsterixAppContextInfo.initialize(appCtx, getNewHyracksClientConnection(), GlobalRecoveryManager.INSTANCE,
+ libraryManager);
proxy = AsterixStateProxy.registerRemoteObject();
appCtx.setDistributedState(proxy);
@@ -107,7 +111,7 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
setupFeedServer(externalProperties);
feedServer.start();
- ExternalLibraryUtils.setUpExternaLibraries(false);
+
ClusterManager.INSTANCE.registerSubscriber(GlobalRecoveryManager.INSTANCE);
ccAppCtx.addClusterLifecycleListener(ClusterLifecycleListener.INSTANCE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index c71d77e..2555b5a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -79,12 +79,11 @@ public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
@Override
public void start(INCApplicationContext ncAppCtx, String[] args) throws Exception {
CmdLineParser parser = new CmdLineParser(this);
-
try {
parser.parseArgument(args);
} catch (CmdLineException e) {
- System.err.println(e.getMessage());
- System.err.println("Usage:");
+ LOGGER.severe(e.getMessage());
+ LOGGER.severe("Usage:");
parser.printUsage(System.err);
throw e;
}
@@ -211,7 +210,8 @@ public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
if (isMetadataNode && !pendingFailbackCompletion) {
runtimeContext.initializeMetadata(systemState == SystemState.NEW_UNIVERSE);
}
- ExternalLibraryUtils.setUpExternaLibraries(isMetadataNode && !pendingFailbackCompletion);
+ ExternalLibraryUtils.setUpExternaLibraries(runtimeContext.getLibraryManager(),
+ isMetadataNode && !pendingFailbackCompletion);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting lifecycle components");
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
index 3416a77..ba34024 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.event.service.AsterixEventServiceUtil;
-import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.test.aql.ITestLibrarian;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -37,6 +37,14 @@ public class TestLibrarian implements ITestLibrarian {
public static final String LIBRARY_DIR_NAME = "library";
+ // The following list includes a library manager for the CC
+ // and library managers for NCs (one-per-NC).
+ private final List<ILibraryManager> libraryManagers;
+
+ public TestLibrarian(List<ILibraryManager> libraryManagers) {
+ this.libraryManagers = libraryManagers;
+ }
+
@Override
public void install(String dvName, String libName, String libPath) throws Exception {
// get the directory of the to be installed libraries
@@ -56,8 +64,10 @@ public class TestLibrarian implements ITestLibrarian {
throw new Exception("Couldn't unzip the file: " + libPath, e);
}
- // for each file (library), register library
- ExternalLibraryUtils.registerLibrary(dvName, libName, true, destinationDir);
+
+ for (ILibraryManager libraryManager : libraryManagers) {
+ ExternalLibraryUtils.registerLibrary(libraryManager, dvName, libName);
+ }
// get library file
// install if needed (add functions, adapters, datasources, parsers to the metadata)
// <Not required for use>
@@ -67,7 +77,9 @@ public class TestLibrarian implements ITestLibrarian {
@Override
public void uninstall(String dvName, String libName) throws RemoteException, AsterixException, ACIDException {
ExternalLibraryUtils.uninstallLibrary(dvName, libName);
- ExternalLibraryManager.deregisterLibraryClassLoader(dvName, libName);
+ for (ILibraryManager libraryManager : libraryManagers) {
+ libraryManager.deregisterLibraryClassLoader(dvName, libName);
+ }
}
public static void removeLibraryDir() throws IOException {
@@ -78,11 +90,16 @@ public class TestLibrarian implements ITestLibrarian {
FileUtils.deleteQuietly(installLibDir);
}
- public static void cleanup() throws AsterixException, RemoteException, ACIDException {
- List<Pair<String, String>> libs = ExternalLibraryManager.getAllLibraries();
- for (Pair<String, String> dvAndLib : libs) {
- ExternalLibraryUtils.uninstallLibrary(dvAndLib.first, dvAndLib.second);
- ExternalLibraryManager.deregisterLibraryClassLoader(dvAndLib.first, dvAndLib.second);
+ public void cleanup() throws AsterixException, RemoteException, ACIDException {
+ for (ILibraryManager libraryManager : libraryManagers) {
+ List<Pair<String, String>> libs = libraryManager.getAllLibraries();
+ for (Pair<String, String> dvAndLib : libs) {
+ ExternalLibraryUtils.uninstallLibrary(dvAndLib.first, dvAndLib.second);
+ libraryManager.deregisterLibraryClassLoader(dvAndLib.first, dvAndLib.second);
+ }
}
+ // get the directory of the to be installed libraries
+ File installLibDir = ExternalLibraryUtils.getLibraryInstallDir();
+ FileUtils.deleteQuietly(installLibDir);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index dfb61c2..baa3923 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -21,10 +21,12 @@ package org.apache.asterix.test.runtime;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import java.util.logging.Logger;
import org.apache.asterix.app.external.TestLibrarian;
import org.apache.asterix.common.config.AsterixTransactionProperties;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.test.aql.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.lang3.StringUtils;
@@ -49,9 +51,10 @@ public class ExecutionTest {
protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
protected static AsterixTransactionProperties txnProperties;
- private static final TestExecutor testExecutor = new TestExecutor();
+ protected static final TestExecutor testExecutor = new TestExecutor();
private static final boolean cleanupOnStart = true;
private static final boolean cleanupOnStop = true;
+ private static TestLibrarian librarian;
@BeforeClass
public static void setUp() throws Exception {
@@ -60,8 +63,9 @@ public class ExecutionTest {
outdir.mkdirs();
// remove library directory
TestLibrarian.removeLibraryDir();
- testExecutor.setLibrarian(new TestLibrarian());
- ExecutionTestUtil.setUp(cleanupOnStart);
+ List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart);
+ librarian = new TestLibrarian(libraryManagers);
+ testExecutor.setLibrarian(librarian);
} catch (Throwable th) {
th.printStackTrace();
throw th;
@@ -102,7 +106,7 @@ public class ExecutionTest {
@Test
public void test() throws Exception {
- TestLibrarian.cleanup();
+ librarian.cleanup();
testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
index d919c92..813def3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
@@ -20,18 +20,21 @@ package org.apache.asterix.test.runtime;
import java.io.File;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
+import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.testframework.xml.TestGroup;
import org.apache.asterix.testframework.xml.TestSuite;
import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
public class ExecutionTestUtil {
@@ -43,7 +46,7 @@ public class ExecutionTestUtil {
protected static TestGroup FailedGroup;
- public static void setUp(boolean cleanup) throws Exception {
+ public static List<ILibraryManager> setUp(boolean cleanup) throws Exception {
System.out.println("Starting setup");
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting setup");
@@ -69,16 +72,17 @@ public class ExecutionTestUtil {
FailedGroup = new TestGroup();
FailedGroup.setName("failed");
- }
- private static void validateBufferCacheState() {
+ List<ILibraryManager> libraryManagers = new ArrayList<>();
+ // Adds the library manager for CC.
+ libraryManagers.add(AsterixAppContextInfo.getInstance().getLibraryManager());
+ // Adds library managers for NCs, one-per-NC.
for (NodeControllerService nc : AsterixHyracksIntegrationUtil.ncs) {
- IAsterixAppRuntimeContext appCtx = (IAsterixAppRuntimeContext) nc.getApplicationContext()
+ IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) nc.getApplicationContext()
.getApplicationObject();
- if (!((BufferCache) appCtx.getBufferCache()).isClean()) {
- throw new IllegalStateException();
- }
+ libraryManagers.add(runtimeCtx.getLibraryManager());
}
+ return libraryManagers;
}
public static void tearDown(boolean cleanup) throws Exception {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
index b577d29..7d39b31 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
@@ -23,8 +23,6 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.Collection;
-import org.apache.asterix.app.external.TestLibrarian;
-import org.apache.asterix.test.aql.TestExecutor;
import org.apache.asterix.test.runtime.RepeatRule.Repeat;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.junit.Rule;
@@ -84,7 +82,6 @@ class RepeatRule implements MethodRule {
public class RepeatedTest extends ExecutionTest {
private int count;
- private final TestExecutor testExecutor = new TestExecutor();
@Parameters(name = "RepeatedTest {index}: {0}")
public static Collection<Object[]> tests() throws Exception {
@@ -94,7 +91,6 @@ public class RepeatedTest extends ExecutionTest {
public RepeatedTest(TestCaseContext tcCtx) {
super(tcCtx);
- testExecutor.setLibrarian(new TestLibrarian());
count = 0;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index ee9ed4a..100da63 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -24,6 +24,7 @@ import java.util.concurrent.Executor;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.replication.IReplicaResourcesManager;
import org.apache.asterix.common.replication.IReplicationChannel;
@@ -83,6 +84,8 @@ public interface IAsterixAppRuntimeContext {
public IReplicationChannel getReplicationChannel();
+ public ILibraryManager getLibraryManager();
+
public void initializeResourceIdFactory() throws HyracksDataException;
/**
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IAsterixApplicationContextInfo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IAsterixApplicationContextInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IAsterixApplicationContextInfo.java
index 9647452..57fb335 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IAsterixApplicationContextInfo.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IAsterixApplicationContextInfo.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common.dataflow;
import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.hyracks.api.application.ICCApplicationContext;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import org.apache.hyracks.storage.common.IStorageManagerInterface;
@@ -50,5 +51,13 @@ public interface IAsterixApplicationContextInfo {
*/
public ICCApplicationContext getCCApplicationContext();
+ /**
+ * @return the global recovery manager.
+ */
public IGlobalRecoveryMaanger getGlobalRecoveryManager();
+
+ /**
+ * @return the library manager (at CC side).
+ */
+ public ILibraryManager getLibraryManager();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
new file mode 100644
index 0000000..5dca73e
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.library;
+
+import java.util.List;
+
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public interface ILibraryManager {
+
+ /**
+ * Registers the library class loader with the external library manager.
+ * <code>dataverseName</code> and <code>libraryName</code> uniquely identifies a class loader.
+ *
+ * @param dataverseName
+ * @param libraryName
+ * @param classLoader
+ */
+ public void registerLibraryClassLoader(String dataverseName, String libraryName, ClassLoader classLoader);
+
+ /**
+ * @return all registered libraries.
+ */
+ public List<Pair<String, String>> getAllLibraries();
+
+ /**
+ * De-registers a library class loader.
+ *
+ * @param dataverseName
+ * @param libraryName
+ */
+ public void deregisterLibraryClassLoader(String dataverseName, String libraryName);
+
+ /**
+ * Finds a class loader for a given pair of dataverse name and library name.
+ *
+ * @param dataverseName
+ * @param libraryName
+ * @return the library class loader associated with the dataverse and library.
+ */
+ public ClassLoader getLibraryClassLoader(String dataverseName, String libraryName);
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 22f046f..f4e67f3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -21,7 +21,9 @@ package org.apache.asterix.external.adapter.factory;
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
@@ -83,8 +85,10 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF
@Override
public synchronized IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition)
throws HyracksDataException {
+ IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
+ .getApplicationContext().getApplicationObject();
try {
- restoreExternalObjects();
+ restoreExternalObjects(runtimeCtx.getLibraryManager());
} catch (AsterixException e) {
throw new HyracksDataException(e);
}
@@ -94,9 +98,8 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF
}
feedLogManager.touch();
}
- IDataFlowController controller =
- DataflowControllerProvider.getDataflowController(recordType, ctx, partition,
- dataSourceFactory, dataParserFactory, configuration, indexingOp, isFeed, feedLogManager);
+ IDataFlowController controller = DataflowControllerProvider.getDataflowController(recordType, ctx, partition,
+ dataSourceFactory, dataParserFactory, configuration, indexingOp, isFeed, feedLogManager);
if (isFeed) {
return new FeedAdapter((AbstractFeedDataFlowController) controller);
} else {
@@ -104,9 +107,9 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF
}
}
- private void restoreExternalObjects() throws AsterixException {
+ private void restoreExternalObjects(ILibraryManager libraryManager) throws AsterixException {
if (dataSourceFactory == null) {
- dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(configuration);
+ dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
// create and configure parser factory
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
@@ -115,7 +118,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF
}
if (dataParserFactory == null) {
// create and configure parser factory
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration);
+ dataParserFactory = ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
@@ -123,17 +126,16 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF
}
@Override
- public void configure(Map<String, String> configuration)
- throws AsterixException {
+ public void configure(ILibraryManager libraryManager, Map<String, String> configuration) throws AsterixException {
this.configuration = configuration;
ExternalDataUtils.validateDataSourceParameters(configuration);
- dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(configuration);
+ dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
}
dataSourceFactory.configure(configuration);
ExternalDataUtils.validateDataParserParameters(configuration);
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration);
+ dataParserFactory = ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
index fa6897f..01fcfc2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/LookupAdapterFactory.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.ILookupReaderFactory;
import org.apache.asterix.external.api.ILookupRecordReader;
import org.apache.asterix.external.api.IRecordDataParser;
@@ -76,10 +77,11 @@ public class LookupAdapterFactory<T> implements Serializable {
}
}
- public void configure(Map<String, String> configuration) throws AsterixException {
+ public void configure(ILibraryManager libraryManager, Map<String, String> configuration) throws AsterixException {
this.configuration = configuration;
readerFactory = LookupReaderFactoryProvider.getLookupReaderFactory(configuration);
- dataParserFactory = (IRecordDataParserFactory<T>) ParserFactoryProvider.getDataParserFactory(configuration);
+ dataParserFactory = (IRecordDataParserFactory<T>) ParserFactoryProvider.getDataParserFactory(libraryManager,
+ configuration);
dataParserFactory.setRecordType(recordType);
readerFactory.configure(configuration);
dataParserFactory.configure(configuration);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
index 505c4b2..2d42ba9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -65,11 +66,11 @@ public interface IAdapterFactory extends Serializable {
public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws HyracksDataException;
/**
+ * @param libraryManager
* @param configuration
* @throws Exception
*/
- public void configure(Map<String, String> configuration)
- throws AsterixException;
+ public void configure(ILibraryManager libraryManager, Map<String, String> configuration) throws AsterixException;
public void setOutputType(ARecordType outputType);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
index 7f416c7..b0594d2 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
@@ -20,7 +20,9 @@ package org.apache.asterix.external.library;
import java.io.IOException;
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IExternalFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.external.api.IFunctionHelper;
@@ -29,6 +31,7 @@ import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
@@ -64,7 +67,17 @@ public abstract class ExternalFunction implements IExternalFunction {
String[] fnameComponents = finfo.getFunctionIdentifier().getName().split("#");
String functionLibary = fnameComponents[0];
String dataverse = finfo.getFunctionIdentifier().getNamespace();
- ClassLoader libraryClassLoader = ExternalLibraryManager.getLibraryClassLoader(dataverse, functionLibary);
+ ILibraryManager libraryManager;
+ if (context == null) {
+ // Gets the library manager for compile-time constant folding.
+ libraryManager = AsterixAppContextInfo.getInstance().getLibraryManager();
+ } else {
+ // Gets the library manager for real runtime evaluation.
+ IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) context.getJobletContext()
+ .getApplicationContext().getApplicationObject();
+ libraryManager = runtimeCtx.getLibraryManager();
+ }
+ ClassLoader libraryClassLoader = libraryManager.getLibraryClassLoader(dataverse, functionLibary);
String classname = finfo.getFunctionBody().trim();
Class<?> clazz;
try {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index 0283bf2..3373f70 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -24,19 +24,15 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.hyracks.algebricks.common.utils.Pair;
-public class ExternalLibraryManager {
+public class ExternalLibraryManager implements ILibraryManager {
- private static final Map<String, ClassLoader> libraryClassLoaders = new HashMap<String, ClassLoader>();
+ private final Map<String, ClassLoader> libraryClassLoaders = new HashMap<>();
- /**
- * Register the library class loader with the external library manager
- * @param dataverseName
- * @param libraryName
- * @param classLoader
- */
- public static void registerLibraryClassLoader(String dataverseName, String libraryName, ClassLoader classLoader) {
+ @Override
+ public void registerLibraryClassLoader(String dataverseName, String libraryName, ClassLoader classLoader) {
String key = getKey(dataverseName, libraryName);
synchronized (libraryClassLoaders) {
if (libraryClassLoaders.get(key) != null) {
@@ -46,7 +42,8 @@ public class ExternalLibraryManager {
}
}
- public static List<Pair<String, String>> getAllLibraries() {
+ @Override
+ public List<Pair<String, String>> getAllLibraries() {
ArrayList<Pair<String, String>> libs = new ArrayList<>();
synchronized (libraryClassLoaders) {
for (Entry<String, ClassLoader> entry : libraryClassLoaders.entrySet()) {
@@ -56,7 +53,8 @@ public class ExternalLibraryManager {
return libs;
}
- public static void deregisterLibraryClassLoader(String dataverseName, String libraryName) {
+ @Override
+ public void deregisterLibraryClassLoader(String dataverseName, String libraryName) {
String key = getKey(dataverseName, libraryName);
synchronized (libraryClassLoaders) {
if (libraryClassLoaders.get(key) != null) {
@@ -65,7 +63,8 @@ public class ExternalLibraryManager {
}
}
- public static ClassLoader getLibraryClassLoader(String dataverseName, String libraryName) {
+ @Override
+ public ClassLoader getLibraryClassLoader(String dataverseName, String libraryName) {
String key = getKey(dataverseName, libraryName);
return libraryClassLoaders.get(key);
}
@@ -75,10 +74,10 @@ public class ExternalLibraryManager {
}
private static Pair<String, String> getDataverseAndLibararyName(String key) {
- int index = key.indexOf(".");
+ int index = key.indexOf('.');
String dataverse = key.substring(0, index);
String library = key.substring(index + 1);
- return new Pair<String, String>(dataverse, library);
+ return new Pair<>(dataverse, library);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index 58e9ea7..69aa59a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -21,11 +21,12 @@ package org.apache.asterix.external.operators;
import java.util.Map;
import java.util.logging.Logger;
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.management.FeedId;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
-import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
@@ -94,24 +95,26 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
if (adaptorFactory == null) {
- try {
- adaptorFactory = createExternalAdapterFactory(ctx, partition);
- } catch (Exception exception) {
- throw new HyracksDataException(exception);
- }
+ adaptorFactory = createExternalAdapterFactory(ctx);
}
return new FeedIntakeOperatorNodePushable(ctx, feedId, adaptorFactory, partition, policyAccessor,
recordDescProvider, this);
}
- private IAdapterFactory createExternalAdapterFactory(IHyracksTaskContext ctx, int partition) throws Exception {
- IAdapterFactory adapterFactory = null;
- ClassLoader classLoader = ExternalLibraryManager.getLibraryClassLoader(feedId.getDataverse(),
- adaptorLibraryName);
+ private IAdapterFactory createExternalAdapterFactory(IHyracksTaskContext ctx) throws HyracksDataException {
+ IAdapterFactory adapterFactory;
+ IAsterixAppRuntimeContext runtimeCtx =
+ (IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
+ ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
+ ClassLoader classLoader = libraryManager.getLibraryClassLoader(feedId.getDataverse(), adaptorLibraryName);
if (classLoader != null) {
- adapterFactory = ((IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance()));
- adapterFactory.setOutputType(adapterOutputType);
- adapterFactory.configure(adaptorConfiguration);
+ try {
+ adapterFactory = (IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
+ adapterFactory.setOutputType(adapterOutputType);
+ adapterFactory.configure(libraryManager, adaptorConfiguration);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
} else {
String message = "Unable to create adapter as class loader not configured for library " + adaptorLibraryName
+ " in dataverse " + feedId.getDataverse();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/RSSParserFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/RSSParserFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/RSSParserFactory.java
index 4f699fc..7465455 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/RSSParserFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/RSSParserFactory.java
@@ -35,6 +35,7 @@ public class RSSParserFactory implements IRecordDataParserFactory<SyndEntryImpl>
@Override
public void configure(Map<String, String> configuration) {
+ // Nothing to be configured.
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java
index f20f802..d6e536d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java
@@ -35,6 +35,7 @@ public class TweetParserFactory implements IRecordDataParserFactory<Status> {
@Override
public void configure(Map<String, String> configuration) {
+ // Nothing to be configured.
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
index 8f6c85f..4cc8e33 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
import org.apache.asterix.external.adapter.factory.LookupAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
@@ -37,18 +38,18 @@ import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
public class AdapterFactoryProvider {
// Adapters
- public static IAdapterFactory getAdapterFactory(String adapterName, Map<String, String> configuration,
- ARecordType itemType, ARecordType metaType) throws AsterixException {
+ public static IAdapterFactory getAdapterFactory(ILibraryManager libraryManager, String adapterName,
+ Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AsterixException {
ExternalDataCompatibilityUtils.prepare(adapterName, configuration);
GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
adapterFactory.setOutputType(itemType);
adapterFactory.setMetaType(metaType);
- adapterFactory.configure(configuration);
+ adapterFactory.configure(libraryManager, configuration);
return adapterFactory;
}
// Indexing Adapters
- public static IIndexingAdapterFactory getIndexingAdapterFactory(String adapterName,
+ public static IIndexingAdapterFactory getIndexingAdapterFactory(ILibraryManager libraryManager, String adapterName,
Map<String, String> configuration, ARecordType itemType, List<ExternalFile> snapshot, boolean indexingOp,
ARecordType metaType) throws AsterixException {
ExternalDataCompatibilityUtils.prepare(adapterName, configuration);
@@ -56,17 +57,17 @@ public class AdapterFactoryProvider {
adapterFactory.setOutputType(itemType);
adapterFactory.setMetaType(metaType);
adapterFactory.setSnapshot(snapshot, indexingOp);
- adapterFactory.configure(configuration);
+ adapterFactory.configure(libraryManager, configuration);
return adapterFactory;
}
// Lookup Adapters
- public static LookupAdapterFactory<?> getLookupAdapterFactory(Map<String, String> configuration,
- ARecordType recordType, int[] ridFields, boolean retainInput, boolean retainMissing,
- IMissingWriterFactory missingWriterFactory) throws AsterixException {
+ public static LookupAdapterFactory<?> getLookupAdapterFactory(ILibraryManager libraryManager,
+ Map<String, String> configuration, ARecordType recordType, int[] ridFields, boolean retainInput,
+ boolean retainMissing, IMissingWriterFactory missingWriterFactory) throws AsterixException {
LookupAdapterFactory<?> adapterFactory = new LookupAdapterFactory<>(recordType, ridFields, retainInput,
retainMissing, missingWriterFactory);
- adapterFactory.configure(configuration);
+ adapterFactory.configure(libraryManager, configuration);
return adapterFactory;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index 452ac6e..ad11171 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.provider;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
@@ -41,24 +42,25 @@ public class DatasourceFactoryProvider {
private DatasourceFactoryProvider() {
}
- public static IExternalDataSourceFactory getExternalDataSourceFactory(Map<String, String> configuration)
- throws AsterixException {
+ public static IExternalDataSourceFactory getExternalDataSourceFactory(ILibraryManager libraryManager,
+ Map<String, String> configuration) throws AsterixException {
if (ExternalDataUtils.getDataSourceType(configuration).equals(DataSourceType.RECORDS)) {
String reader = configuration.get(ExternalDataConstants.KEY_READER);
- return DatasourceFactoryProvider.getRecordReaderFactory(reader, configuration);
+ return DatasourceFactoryProvider.getRecordReaderFactory(libraryManager, reader, configuration);
} else {
// get stream source
String streamSource = configuration.get(ExternalDataConstants.KEY_STREAM_SOURCE);
- return DatasourceFactoryProvider.getInputStreamFactory(streamSource, configuration);
+ return DatasourceFactoryProvider.getInputStreamFactory(libraryManager, streamSource, configuration);
}
}
- public static IInputStreamFactory getInputStreamFactory(String streamSource, Map<String, String> configuration)
- throws AsterixException {
+ public static IInputStreamFactory getInputStreamFactory(ILibraryManager libraryManager, String streamSource,
+ Map<String, String> configuration) throws AsterixException {
IInputStreamFactory streamSourceFactory;
if (ExternalDataUtils.isExternal(streamSource)) {
String dataverse = ExternalDataUtils.getDataverse(configuration);
- streamSourceFactory = ExternalDataUtils.createExternalInputStreamFactory(dataverse, streamSource);
+ streamSourceFactory = ExternalDataUtils.createExternalInputStreamFactory(libraryManager, dataverse,
+ streamSource);
} else {
switch (streamSource) {
case ExternalDataConstants.STREAM_LOCAL_FILESYSTEM:
@@ -85,10 +87,10 @@ public class DatasourceFactoryProvider {
return streamSourceFactory;
}
- public static IRecordReaderFactory<?> getRecordReaderFactory(String reader, Map<String, String> configuration)
- throws AsterixException {
+ public static IRecordReaderFactory<?> getRecordReaderFactory(ILibraryManager libraryManager, String reader,
+ Map<String, String> configuration) throws AsterixException {
if (reader.equals(ExternalDataConstants.EXTERNAL)) {
- return ExternalDataUtils.createExternalRecordReaderFactory(configuration);
+ return ExternalDataUtils.createExternalRecordReaderFactory(libraryManager, configuration);
}
switch (reader) {
case ExternalDataConstants.READER_HDFS:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
index b37198a..e98e744 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
@@ -23,6 +23,7 @@ import java.util.Map;
import javax.annotation.Nonnull;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.parser.factory.ADMDataParserFactory;
import org.apache.asterix.external.parser.factory.DelimitedDataParserFactory;
@@ -38,12 +39,13 @@ public class ParserFactoryProvider {
private ParserFactoryProvider() {
}
- public static IDataParserFactory getDataParserFactory(Map<String, String> configuration) throws AsterixException {
+ public static IDataParserFactory getDataParserFactory(ILibraryManager libraryManager,
+ Map<String, String> configuration) throws AsterixException {
IDataParserFactory parserFactory;
String parserFactoryName = configuration.get(ExternalDataConstants.KEY_DATA_PARSER);
if ((parserFactoryName != null) && ExternalDataUtils.isExternal(parserFactoryName)) {
- return ExternalDataUtils.createExternalParserFactory(ExternalDataUtils.getDataverse(configuration),
- parserFactoryName);
+ return ExternalDataUtils.createExternalParserFactory(libraryManager,
+ ExternalDataUtils.getDataverse(configuration), parserFactoryName);
} else {
String parserFactoryKey = ExternalDataUtils.getRecordFormat(configuration);
if (parserFactoryKey == null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 69882d0..19781f9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -22,11 +22,11 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
-import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
@@ -114,8 +114,8 @@ public class ExternalDataUtils {
&& (aString.trim().length() > 1));
}
- public static ClassLoader getClassLoader(String dataverse, String library) {
- return ExternalLibraryManager.getLibraryClassLoader(dataverse, library);
+ public static ClassLoader getClassLoader(ILibraryManager libraryManager, String dataverse, String library) {
+ return libraryManager.getLibraryClassLoader(dataverse, library);
}
public static String getLibraryName(String aString) {
@@ -126,12 +126,12 @@ public class ExternalDataUtils {
return aString.trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[1];
}
- public static IInputStreamFactory createExternalInputStreamFactory(String dataverse, String stream)
- throws AsterixException {
+ public static IInputStreamFactory createExternalInputStreamFactory(ILibraryManager libraryManager, String dataverse,
+ String stream) throws AsterixException {
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
- ClassLoader classLoader = getClassLoader(dataverse, libraryName);
+ ClassLoader classLoader = getClassLoader(libraryManager, dataverse, libraryName);
return ((IInputStreamFactory) (classLoader.loadClass(className).newInstance()));
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new AsterixException("Failed to create stream factory", e);
@@ -210,8 +210,8 @@ public class ExternalDataUtils {
return false;
}
- public static IRecordReaderFactory<?> createExternalRecordReaderFactory(Map<String, String> configuration)
- throws AsterixException {
+ public static IRecordReaderFactory<?> createExternalRecordReaderFactory(ILibraryManager libraryManager,
+ Map<String, String> configuration) throws AsterixException {
String readerFactory = configuration.get(ExternalDataConstants.KEY_READER_FACTORY);
if (readerFactory == null) {
throw new AsterixException("to use " + ExternalDataConstants.EXTERNAL + " reader, the parameter "
@@ -228,8 +228,7 @@ public class ExternalDataUtils {
+ " must follow the format \"DataverseName.LibraryName#ReaderFactoryFullyQualifiedName\"");
}
- ClassLoader classLoader = ExternalLibraryManager.getLibraryClassLoader(dataverseAndLibrary[0],
- dataverseAndLibrary[1]);
+ ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverseAndLibrary[0], dataverseAndLibrary[1]);
try {
return (IRecordReaderFactory<?>) classLoader.loadClass(libraryAndFactory[1]).newInstance();
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
@@ -237,12 +236,12 @@ public class ExternalDataUtils {
}
}
- public static IDataParserFactory createExternalParserFactory(String dataverse, String parserFactoryName)
- throws AsterixException {
+ public static IDataParserFactory createExternalParserFactory(ILibraryManager libraryManager, String dataverse,
+ String parserFactoryName) throws AsterixException {
try {
String library = parserFactoryName.substring(0,
parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR));
- ClassLoader classLoader = ExternalLibraryManager.getLibraryClassLoader(dataverse, library);
+ ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverse, library);
return (IDataParserFactory) classLoader
.loadClass(parserFactoryName
.substring(parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR) + 1))
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index e029c09..6850c2b 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.IAsterixPropertiesProvider;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
@@ -77,11 +78,9 @@ public class TestTypedAdapterFactory implements IAdapterFactory {
ADMDataParser parser;
ITupleForwarder forwarder;
ArrayTupleBuilder tb;
- IAsterixPropertiesProvider propertiesProvider =
- (IAsterixPropertiesProvider) ((NodeControllerService) ctx
- .getJobletContext().getApplicationContext().getControllerService())
- .getApplicationContext()
- .getApplicationObject();
+ IAsterixPropertiesProvider propertiesProvider = (IAsterixPropertiesProvider) ((NodeControllerService) ctx
+ .getJobletContext().getApplicationContext().getControllerService()).getApplicationContext()
+ .getApplicationObject();
ClusterPartition nodePartition = propertiesProvider.getMetadataProperties().getNodePartitions()
.get(nodeId)[0];
try {
@@ -127,7 +126,7 @@ public class TestTypedAdapterFactory implements IAdapterFactory {
}
@Override
- public void configure(Map<String, String> configuration) {
+ public void configure(ILibraryManager context, Map<String, String> configuration) {
this.configuration = configuration;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/factory/TestRecordWithPKParserFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/factory/TestRecordWithPKParserFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/factory/TestRecordWithPKParserFactory.java
index dae8a8f..5b23094 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/factory/TestRecordWithPKParserFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/factory/TestRecordWithPKParserFactory.java
@@ -50,7 +50,8 @@ public class TestRecordWithPKParserFactory<T> implements IRecordDataParserFactor
TreeMap<String, String> parserConf = new TreeMap<String, String>();
format = configuration.get(ExternalDataConstants.KEY_RECORD_FORMAT);
parserConf.put(ExternalDataConstants.KEY_FORMAT, format);
- recordParserFactory = (IRecordDataParserFactory<char[]>) ParserFactoryProvider.getDataParserFactory(parserConf);
+ recordParserFactory =
+ (IRecordDataParserFactory<char[]>) ParserFactoryProvider.getDataParserFactory(null, parserConf);
recordParserFactory.setRecordType(recordType);
recordParserFactory.configure(configuration);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f09607dd/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 226ab3c..a5e5cda 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -42,6 +42,7 @@ import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactor
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.common.utils.StoragePathUtil;
@@ -194,6 +195,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
private boolean isTemporaryDatasetWriteJob = true;
private final AsterixStorageProperties storageProperties;
+ private final ILibraryManager libraryManager;
public String getPropertyValue(String propertyName) {
return config.get(propertyName);
@@ -215,6 +217,11 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
this.defaultDataverse = defaultDataverse;
this.stores = AsterixAppContextInfo.getInstance().getMetadataProperties().getStores();
this.storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
+ this.libraryManager = AsterixAppContextInfo.getInstance().getLibraryManager();
+ }
+
+ public ILibraryManager getLibraryManager() {
+ return libraryManager;
}
public void setJobId(JobId jobId) {
@@ -482,8 +489,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
List<List<String>> primaryKeys, ARecordType metaType) throws AlgebricksException {
try {
configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName());
- IAdapterFactory adapterFactory =
- AdapterFactoryProvider.getAdapterFactory(adapterName, configuration, itemType, metaType);
+ IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(libraryManager, adapterName,
+ configuration, itemType, metaType);
// check to see if dataset is indexed
Index filesIndex =
@@ -535,7 +542,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime(
JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception {
Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> factoryOutput = null;
- factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx);
+ factoryOutput =
+ FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx, libraryManager);
ARecordType recordType = FeedMetadataUtil.getOutputType(primaryFeed, primaryFeed.getAdapterConfiguration(),
ExternalDataConstants.KEY_TYPE_NAME);
IAdapterFactory adapterFactory = factoryOutput.first;
@@ -2080,8 +2088,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
- public Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
- splitProviderAndPartitionConstraintsForDataverse(String dataverse) {
+ public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataverse(
+ String dataverse) {
return SplitsAndConstraintsUtil.splitProviderAndPartitionConstraintsForDataverse(dataverse);
}
@@ -2104,8 +2112,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
public Dataset findDataset(String dataverse, String dataset) throws AlgebricksException {
- String dv = dataverse == null ? (defaultDataverse == null ? null : defaultDataverse.getDataverseName())
- : dataverse;
+ String dv =
+ dataverse == null ? (defaultDataverse == null ? null : defaultDataverse.getDataverseName()) : dataverse;
if (dv == null) {
return null;
}
@@ -2195,7 +2203,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
this.locks = locks;
}
- public static Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDataLookupRuntime(
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDataLookupRuntime(
JobSpecification jobSpec, Dataset dataset, Index secondaryIndex, int[] ridIndexes, boolean retainInput,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> outputVars, IOperatorSchema opSchema,
JobGenContext context, AqlMetadataProvider metadataProvider, boolean retainMissing)
@@ -2209,7 +2217,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
// Create the adapter factory <- right now there is only one. if there are more in the future, we can create
// a map->
ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
- LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getLookupAdapterFactory(
+ LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getLookupAdapterFactory(libraryManager,
datasetDetails.getProperties(), (ARecordType) itemType, ridIndexes, retainInput, retainMissing,
context.getMissingWriterFactory());
@@ -2349,9 +2357,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ITypeTraits[] outputTypeTraits =
new ITypeTraits[recordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
- ISerializerDeserializer[] outputSerDes =
- new ISerializerDeserializer[recordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1)
- + numFilterFields];
+ ISerializerDeserializer[] outputSerDes = new ISerializerDeserializer[recordDesc.getFieldCount()
+ + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
for (int j = 0; j < recordDesc.getFieldCount(); j++) {
outputTypeTraits[j] = recordDesc.getTypeTraits()[j];
outputSerDes[j] = recordDesc.getFields()[j];