You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2020/04/23 09:45:25 UTC
[asterixdb] branch master updated: [NO ISSUE] Fix UDF Metadata
Transactions
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new f2c9308 [NO ISSUE] Fix UDF Metadata Transactions
f2c9308 is described below
commit f2c93085c81628eb9bc7e563259eaf9e4fe7a3bc
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Wed Apr 22 01:43:28 2020 -0700
[NO ISSUE] Fix UDF Metadata Transactions
- Move UDF Metadata Transactions to be initiated from the CC
directly
- Remove old UDF initialization code
- Make DROP DATAVERSE remove libraries
- Make NCs properly read libraries on init with multipart dataverse
names
Change-Id: Ibda23a2e8308937f343d80eff04ede9a235da0d2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4923
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../asterix/api/http/server/UdfApiServlet.java | 166 ++++++++++++---
.../asterix/app/external/ExternalLibraryUtils.java | 225 +++------------------
.../asterix/app/message/AbstractUdfMessage.java | 8 +-
.../asterix/app/message/DeleteUdfMessage.java | 6 +-
.../apache/asterix/app/message/LoadUdfMessage.java | 4 +-
.../app/nc/task/ExternalLibrarySetupTask.java | 2 +-
.../asterix/app/translator/QueryTranslator.java | 14 ++
.../asterix/hyracks/bootstrap/CCApplication.java | 2 +-
.../asterix/app/external/ExternalUDFLibrarian.java | 16 +-
.../deterministic.5.ddl.sqlpp} | 0
.../getCapital.4.ddl.sqlpp} | 0
.../getCapital_open.4.ddl.sqlpp} | 0
.../getCapital_open/getCapital_open.4.lib.sqlpp | 19 --
...ay_sum.5.ddl.sqlpp => my_array_sum.4.ddl.sqlpp} | 0
.../my_array_sum/my_array_sum.4.lib.sqlpp | 19 --
.../external-library/mysum/mysum.7.lib.sqlpp | 19 --
.../mysum_dropinuse.0.ddl.sqlpp} | 3 +-
.../mysum_dropinuse.1.lib.sqlpp} | 2 +-
.../mysum_dropinuse.2.ddl.sqlpp} | 3 +-
.../mysum_dropinuse.3.query.sqlpp} | 3 +-
.../mysum_dropinuse.4.lib.sqlpp} | 0
.../mysum_dropinuse.5.ddl.sqlpp} | 3 +-
.../mysum_dropinuse.6.lib.sqlpp} | 0
.../mysum_dropinuse.7.ddl.sqlpp} | 0
...ion.5.ddl.sqlpp => type_validation.4.ddl.sqlpp} | 0
.../udf_metadata.4.ddl.sqlpp} | 0
.../upperCase.4.ddl.sqlpp} | 2 +-
...lpp => feed-with-external-function.6.ddl.sqlpp} | 0
.../feed-with-external-function.6.lib.sqlpp | 19 --
.../mysum_dropinuse/mysum_dropinuse.1.adm | 1 +
.../resources/runtimets/testsuite_it_sqlpp.xml | 6 +
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../org/apache/asterix/metadata/MetadataNode.java | 5 +
34 files changed, 220 insertions(+), 329 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
index 54a62df..0f5d542 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
@@ -19,21 +19,34 @@
package org.apache.asterix.api.http.server;
import java.io.File;
+import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import org.apache.asterix.app.external.ExternalLibraryUtils;
-import org.apache.asterix.app.message.DeleteUdfMessage;
import org.apache.asterix.app.message.LoadUdfMessage;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
+import org.apache.asterix.common.metadata.LockList;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.DatasourceAdapter;
+import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.metadata.entities.Library;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.deployment.DeploymentId;
@@ -75,6 +88,8 @@ public class UdfApiServlet extends BasicAuthServlet {
@Override
protected void post(IServletRequest request, IServletResponse response) {
+
+ PrintWriter responseWriter = response.writer();
FullHttpRequest req = request.getHttpRequest();
Pair<String, DataverseName> resourceNames;
try {
@@ -85,8 +100,17 @@ public class UdfApiServlet extends BasicAuthServlet {
}
String resourceName = resourceNames.first;
DataverseName dataverse = resourceNames.second;
+ IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
+ MetadataTransactionContext mdTxnCtx = null;
+ LockList mdLockList = null;
File udf = null;
try {
+ MetadataManager.INSTANCE.init();
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null);
+ mdLockList = metadataProvider.getLocks();
+ mdLockUtil.createLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse,
+ resourceName);
File workingDir = new File(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
UDF_TMP_DIR_PREFIX);
if (!workingDir.exists()) {
@@ -102,35 +126,104 @@ public class UdfApiServlet extends BasicAuthServlet {
fc.write(content);
}
}
- IHyracksClientConnection hcc = appCtx.getHcc();
- DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName));
- ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName);
- if (cl != null) {
- deleteUdf(dataverse, resourceName);
- }
- hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true);
- ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(), false,
- FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
- "applications", udfName.toString()));
-
- long reqId = broker.newRequestId();
- List<INcAddressedMessage> requests = new ArrayList<>();
- List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
- ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId)));
- broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+ setupBinariesAndClassloaders(dataverse, resourceName, udf);
+ installLibrary(mdTxnCtx, dataverse, resourceName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
+ try {
+ ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ }
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ responseWriter.write(e.getMessage());
+ responseWriter.flush();
LOGGER.error(e);
- return;
- } finally {
+ if (mdTxnCtx != null) {
+ try {
+ MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+ } catch (RemoteException r) {
+ LOGGER.error("Unable to abort metadata transaction", r);
+ }
+ }
if (udf != null) {
udf.delete();
}
+ return;
+ } finally {
+ if (mdLockList != null) {
+ mdLockList.unlock();
+ }
}
response.setStatus(HttpResponseStatus.OK);
}
+ private void setupBinariesAndClassloaders(DataverseName dataverse, String resourceName, File udf) throws Exception {
+ IHyracksClientConnection hcc = appCtx.getHcc();
+ DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName));
+ ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName);
+ if (cl != null) {
+ //prepare to replace the binary
+ ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+ }
+ hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true);
+ //setup for CC
+ ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(),
+ FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
+ "applications", udfName.toString()));
+ //setup NCs
+ long reqId = broker.newRequestId();
+ List<INcAddressedMessage> requests = new ArrayList<>();
+ List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
+ ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId)));
+ broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+ }
+
+ private static void installLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName)
+ throws RemoteException, AlgebricksException {
+ Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
+ // Get the dataverse
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
+ if (dv == null) {
+ throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE);
+ }
+ if (libraryInMetadata != null) {
+ //replacing binary, library already exists
+ return;
+ }
+ // Add library
+ MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Added library " + libraryName + " to Metadata");
+ }
+ }
+
+ private static void deleteLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName)
+ throws RemoteException, AlgebricksException {
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
+ if (dv == null) {
+ throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE);
+ }
+ Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
+ if (library == null) {
+ throw new AsterixException(ErrorCode.UNKNOWN_LIBRARY);
+ }
+ List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
+ for (Function function : functions) {
+ if (libraryName.equals(function.getLibrary())) {
+ throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName);
+ }
+ }
+ List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
+ for (DatasourceAdapter adapter : adapters) {
+ if (libraryName.equals(adapter.getLibrary())) {
+ throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName);
+ }
+ }
+ MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
+ }
+
public static String makeDeploymentId(DataverseName dv, String resourceName) {
List<String> dvParts = dv.getParts();
dvParts.add(resourceName);
@@ -138,16 +231,6 @@ public class UdfApiServlet extends BasicAuthServlet {
return dvWithLibrarySuffix.getCanonicalForm();
}
- private void deleteUdf(DataverseName dataverse, String resourceName) throws Exception {
- long reqId = broker.newRequestId();
- List<INcAddressedMessage> requests = new ArrayList<>();
- List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
- ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverse, resourceName, reqId)));
- broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
- appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverse, resourceName);
- appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverse, resourceName)));
- }
-
@Override
protected void delete(IServletRequest request, IServletResponse response) {
Pair<String, DataverseName> resourceNames;
@@ -157,14 +240,37 @@ public class UdfApiServlet extends BasicAuthServlet {
response.setStatus(HttpResponseStatus.BAD_REQUEST);
return;
}
+ PrintWriter responseWriter = response.writer();
String resourceName = resourceNames.first;
DataverseName dataverse = resourceNames.second;
+ IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
+ MetadataTransactionContext mdTxnCtx = null;
+ LockList mdLockList = null;
try {
- deleteUdf(dataverse, resourceName);
+ MetadataManager.INSTANCE.init();
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null);
+ mdLockList = metadataProvider.getLocks();
+ mdLockUtil.dropLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse,
+ resourceName);
+ deleteLibrary(mdTxnCtx, dataverse, resourceName);
+ ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
+ try {
+ MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+ } catch (RemoteException r) {
+ LOGGER.error("Unable to abort metadata transaction", r);
+ }
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ responseWriter.write(e.getMessage());
+ responseWriter.flush();
LOGGER.error(e);
return;
+ } finally {
+ if (mdLockList != null) {
+ mdLockList.unlock();
+ }
}
response.setStatus(HttpResponseStatus.OK);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 758bd45..47a460e 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -18,31 +18,23 @@
*/
package org.apache.asterix.app.external;
+import static org.apache.asterix.api.http.server.UdfApiServlet.UDF_RESPONSE_TIMEOUT;
+import static org.apache.asterix.api.http.server.UdfApiServlet.makeDeploymentId;
+
import java.io.File;
import java.io.FilenameFilter;
-import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.rmi.RemoteException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.app.message.DeleteUdfMessage;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.Dataverse;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.Library;
-import org.apache.asterix.metadata.utils.MetadataUtil;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-import org.apache.logging.log4j.Level;
+import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -54,30 +46,17 @@ public class ExternalLibraryUtils {
private ExternalLibraryUtils() {
}
- public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, boolean isMetadataNode,
- String libraryPath) throws Exception {
- // start by un-installing removed libraries (Metadata Node only)
- Map<DataverseName, List<String>> uninstalledLibs = null;
- if (isMetadataNode) {
- uninstalledLibs = uninstallLibraries();
- }
-
- // get the directory of the to be installed libraries
- String[] pathSplit = libraryPath.split("\\.");
- String[] dvSplit = pathSplit[pathSplit.length - 2].split("/");
- DataverseName dataverse = DataverseName.createSinglePartName(dvSplit[dvSplit.length - 1]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
- String name = pathSplit[pathSplit.length - 1].trim();
- File installLibDir = new File(libraryPath);
-
- // directory exists?
- if (installLibDir.exists()) {
- registerClassLoader(externalLibraryManager, dataverse, name, libraryPath);
- configureLibrary(externalLibraryManager, dataverse, name, installLibDir, uninstalledLibs, isMetadataNode);
- }
+ public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, String libraryPath)
+ throws Exception {
+ // get the installed library dirs
+ String[] parts = libraryPath.split(File.separator);
+ DataverseName catenatedDv = DataverseName.createFromCanonicalForm(parts[parts.length - 1]);
+ String libraryName = catenatedDv.getParts().get(catenatedDv.getParts().size() - 1);
+ DataverseName dvName = DataverseName.create(catenatedDv.getParts(), 0, catenatedDv.getParts().size() - 1);
+ registerClassLoader(externalLibraryManager, dvName, libraryName, libraryPath);
}
- public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, boolean isMetadataNode,
- File appDir) throws Exception {
+ public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, File appDir) throws Exception {
File[] libs = appDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -86,170 +65,20 @@ public class ExternalLibraryUtils {
});
if (libs != null) {
for (File lib : libs) {
- setUpExternaLibrary(externalLibraryManager, isMetadataNode, lib.getAbsolutePath());
- }
- }
- }
-
- /**
- * un-install libraries.
- *
- * @return a map from dataverse -> list of uninstalled libraries.
- * @throws Exception
- */
- private static Map<DataverseName, List<String>> uninstallLibraries() throws Exception {
- Map<DataverseName, List<String>> uninstalledLibs = new HashMap<>();
- // get the directory of the un-install libraries
- File uninstallLibDir = getLibraryUninstallDir();
- String[] uninstallLibNames;
- // directory exists?
- if (uninstallLibDir.exists()) {
- // list files
- uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter);
- for (String uninstallLibName : uninstallLibNames) {
- // Get the <dataverse name - library name> pair
- String[] components = uninstallLibName.split("\\.");
- DataverseName dataverse = DataverseName.createSinglePartName(components[0]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
- String libName = components[1];
- // un-install
- uninstallLibrary(dataverse, libName);
- // delete the library file
- new File(uninstallLibDir, uninstallLibName).delete();
- // add the library to the list of uninstalled libraries
- List<String> uinstalledLibsInDv = uninstalledLibs.get(dataverse);
- if (uinstalledLibsInDv == null) {
- uinstalledLibsInDv = new ArrayList<>();
- uninstalledLibs.put(dataverse, uinstalledLibsInDv);
- }
- uinstalledLibsInDv.add(libName);
+ setUpExternaLibrary(externalLibraryManager, lib.getAbsolutePath());
}
}
- return uninstalledLibs;
}
- /**
- * Remove the library from metadata completely.
- * TODO Currently, external libraries only include functions and adapters. we need to extend this to include:
- * 1. external data source
- * 2. data parser
- *
- * @param dataverse
- * @param libraryName
- * @return true if the library was found and removed, false otherwise
- * @throws AsterixException
- * @throws RemoteException
- * @throws ACIDException
- */
- public static boolean uninstallLibrary(DataverseName dataverse, String libraryName)
- throws AsterixException, RemoteException, ACIDException {
- MetadataTransactionContext mdTxnCtx = null;
- try {
- // begin transaction
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- // make sure dataverse exists
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
- if (dv == null) {
- return false;
- }
- // make sure library exists
- Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
- if (library == null) {
- return false;
- }
-
- // get dataverse functions
- List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
- for (Function function : functions) {
- // does function belong to library?
- if (function.getName().startsWith(libraryName + "#")) {
- // drop the function
- MetadataManager.INSTANCE.dropFunction(mdTxnCtx,
- new FunctionSignature(dataverse, function.getName(), function.getArity()));
- }
- }
-
- // get the dataverse adapters
- List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
- for (DatasourceAdapter adapter : adapters) {
- // belong to the library?
- if (adapter.getAdapterIdentifier().getName().startsWith(libraryName + "#")) {
- // remove adapter <! we didn't check if there are feeds which use this adapter>
- MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName());
- }
- }
- // drop the library itself
- MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
- throw new AsterixException(e);
- }
- return true;
- }
-
- private static void addLibraryToMetadata(Map<DataverseName, List<String>> uninstalledLibs, DataverseName dataverse,
- String libraryName) throws ACIDException, RemoteException {
- // Modify metadata accordingly
- List<String> uninstalledLibsInDv = uninstalledLibs.get(dataverse);
- // was this library just un-installed?
- boolean wasUninstalled = uninstalledLibsInDv != null && uninstalledLibsInDv.contains(libraryName);
- MetadataTransactionContext mdTxnCtx = null;
- try {
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
- if (libraryInMetadata != null && !wasUninstalled) {
- // exists in metadata and was not un-installed, we return.
- // Another place which shows that our metadata transactions are broken
- // (we didn't call commit before!!!)
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- }
-
- // Add library
- MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Added library " + libraryName + " to Metadata");
- }
-
- // Get the dataverse
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
- if (dv == null) {
- MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse,
- NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, MetadataUtil.PENDING_NO_OP));
- }
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled()) {
- LOGGER.log(Level.ERROR, "Exception in installing library " + libraryName, e);
- }
- MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
- }
- }
-
- /**
- * Each element of a library is installed as part of a transaction. Any
- * failure in installing an element does not effect installation of other
- * libraries.
- */
- protected static void configureLibrary(ILibraryManager libraryManager, DataverseName dataverse, String libraryName,
- final File libraryDir, Map<DataverseName, List<String>> uninstalledLibs, boolean isMetadataNode)
- throws Exception {
-
- String[] libraryDescriptors = libraryDir.list((dir, name) -> name.endsWith(".xml"));
-
- if (libraryDescriptors == null) {
- throw new IOException("Unable to list files in directory " + libraryDir);
- }
-
- if (libraryDescriptors.length > 1) {
- throw new IllegalStateException("More than 1 library descriptors defined");
- }
-
- // Prepare possible parameters
- if (isMetadataNode) {
- addLibraryToMetadata(uninstalledLibs, dataverse, libraryName);
- }
+ public static void deleteDeployedUdf(ICCMessageBroker broker, ICcApplicationContext appCtx,
+ DataverseName dataverseName, String lib) throws Exception {
+ long reqId = broker.newRequestId();
+ List<INcAddressedMessage> requests = new ArrayList<>();
+ List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
+ ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverseName, lib, reqId)));
+ broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+ appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverseName, lib);
+ appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverseName, lib)));
}
/**
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
index 1c212fe..b8c77e7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
@@ -46,12 +46,9 @@ public abstract class AbstractUdfMessage extends CcIdentifiedMessage implements
@Override
public void handle(INcApplicationContext appCtx) {
ILibraryManager mgr = appCtx.getLibraryManager();
- String mdNodeName = appCtx.getMetadataProperties().getMetadataNodeName();
- String nodeName = appCtx.getServiceContext().getNodeId();
INCMessageBroker broker = (INCMessageBroker) appCtx.getServiceContext().getMessageBroker();
- boolean isMdNode = mdNodeName.equals(nodeName);
try {
- handleAction(mgr, isMdNode, appCtx);
+ handleAction(mgr, appCtx);
broker.sendMessageToCC(getCcId(), new UdfResponseMessage(reqId, null));
} catch (Exception e) {
try {
@@ -64,7 +61,6 @@ public abstract class AbstractUdfMessage extends CcIdentifiedMessage implements
}
- protected abstract void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx)
- throws Exception;
+ protected abstract void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
index 3fb58fa..7bcfa9b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.app.message;
-import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
@@ -32,10 +31,7 @@ public class DeleteUdfMessage extends AbstractUdfMessage {
}
@Override
- protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception {
- if (isMdNode) {
- ExternalLibraryUtils.uninstallLibrary(dataverseName, libraryName);
- }
+ protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) {
mgr.deregisterLibraryClassLoader(dataverseName, libraryName);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
index 6dffb28..7ccdfd4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
@@ -33,8 +33,8 @@ public class LoadUdfMessage extends AbstractUdfMessage {
}
@Override
- protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception {
- ExternalLibraryUtils.setUpExternaLibrary(mgr, isMdNode,
+ protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception {
+ ExternalLibraryUtils.setUpExternaLibrary(mgr,
FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
"applications", dataverseName.getCanonicalForm() + "." + libraryName)); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
index 1ca2b78..40262ac 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
@@ -38,7 +38,7 @@ public class ExternalLibrarySetupTask implements INCLifecycleTask {
public void perform(CcId ccId, IControllerService cs) throws HyracksDataException {
INcApplicationContext appContext = (INcApplicationContext) cs.getApplicationContext();
try {
- ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(), metadataNode,
+ ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(),
cs.getContext().getServerCtx().getAppDir());
} catch (Exception e) {
throw HyracksDataException.create(e);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 16ca37f..814c96e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -50,6 +50,7 @@ import org.apache.asterix.api.http.server.ApiServlet;
import org.apache.asterix.app.active.ActiveEntityEventsListener;
import org.apache.asterix.app.active.ActiveNotificationHandler;
import org.apache.asterix.app.active.FeedEventsListener;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.app.result.ExecutionError;
import org.apache.asterix.app.result.ResultHandle;
import org.apache.asterix.app.result.ResultReader;
@@ -78,6 +79,7 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.utils.JobUtils;
@@ -1307,6 +1309,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
List<JobSpecification> jobsToExecute = new ArrayList<>();
+ List<Library> librariesToDelete = new ArrayList<>();
+ ICCMessageBroker broker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker();
try {
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
if (dv == null) {
@@ -1326,6 +1330,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
function.getDataverseName() + "." + function.getName() + "@" + function.getArity());
}
}
+
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
// # disconnect all feeds from any datasets in the dataverse.
@@ -1381,6 +1386,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
jobsToExecute.add(DataverseUtil.dropDataverseJobSpec(dv, metadataProvider));
+ librariesToDelete = MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx, dataverseName);
// #. mark PendingDropOp on the dataverse record by
// first, deleting the dataverse record from the DATAVERSE_DATASET
// second, inserting the dataverse record with the PendingDropOp value into the
@@ -1397,6 +1403,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
runJob(hcc, jobSpec);
}
+ for (Library lib : librariesToDelete) {
+ ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName());
+ }
+
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -1434,6 +1444,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
for (JobSpecification jobSpec : jobsToExecute) {
runJob(hcc, jobSpec);
}
+
+ for (Library lib : librariesToDelete) {
+ ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName());
+ }
} catch (Exception e2) {
// do no throw exception since still the metadata needs to be compensated.
e.addSuppressed(e2);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index bc10d4f..ae85381 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -157,7 +157,7 @@ public class CCApplication extends BaseCCApplication {
ReplicationProperties repProp =
new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
- ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, false, ccServiceCtx.getServerCtx().getAppDir());
+ ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, ccServiceCtx.getServerCtx().getAppDir());
componentProvider = new StorageComponentProvider();
ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()));
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index fd40a01..0c3ac81 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.URL;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -39,7 +40,6 @@ import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksException;
@SuppressWarnings("squid:S134")
public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
@@ -75,7 +75,7 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
HttpResponse response = hc.execute(post, hcCtx);
response.getEntity().consumeContent();
if (response.getStatusLine().getStatusCode() != 200) {
- throw new HyracksException(response.getStatusLine().toString());
+ throw new AsterixException(response.getStatusLine().toString());
}
}
@@ -93,9 +93,17 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
hcCtx.setAuthCache(ac);
HttpDelete del = new HttpDelete(url.toString());
HttpResponse response = hc.execute(del, hcCtx);
+ String resp = null;
+ int respCode = response.getStatusLine().getStatusCode();
+ if (respCode == 500) {
+ resp = IOUtils.toString(response.getEntity().getContent());
+ }
response.getEntity().consumeContent();
- if (response.getStatusLine().getStatusCode() != 200) {
- throw new AsterixException(response.getStatusLine().toString());
+ if (resp == null && respCode != 200) {
+ resp = response.getStatusLine().toString();
+ }
+ if (resp != null) {
+ throw new AsterixException(resp);
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
similarity index 91%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
index 172bed4..76cc70d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
@@ -16,4 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-uninstall externallibtest testlib admin admin
\ No newline at end of file
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
similarity index 88%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
index 41880cb..3dc6eb6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-uninstall externallibtest testlib admin admin
+install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
index e44b339..fd815d1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
@@ -16,5 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
+ USE externallibtest;
-uninstall externallibtest testlib admin admin
+create function mysum(a: int32, b: int32) returns int32 language java as "testlib","org.apache.asterix.external.library.MySumFactory";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
similarity index 95%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
index 2b27030..a23c197 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
@@ -16,5 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
+use externallibtest;
-DROP DATAVERSE externallibtest;
+mysum(9.7, 4);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
similarity index 92%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
index e44b339..f8e2ae7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
@@ -16,5 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
+ USE externallibtest;
-uninstall externallibtest testlib admin admin
+drop function externallibtest.mysum@2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
similarity index 96%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
index 65733e4..1a46de9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-DROP DATAVERSE externallibtest;
+ drop dataverse externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
deleted file mode 100644
index 6b2fb1f..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall udfs testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm
new file mode 100644
index 0000000..ca7bf83
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm
@@ -0,0 +1 @@
+13
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 91d4969..b6cf0cc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -44,6 +44,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
+ <compilation-unit name="mysum_dropinuse">
+ <output-dir compare="Text">mysum_dropinuse</output-dir>
+ <expected-error>Library testlib is being used. It cannot be dropped</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
<compilation-unit name="my_array_sum">
<output-dir compare="Text">my_array_sum</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 3d30360..7fef69e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -314,6 +314,7 @@ public class ErrorCode {
public static final int FAILED_TO_PARSE_METADATA = 3115;
public static final int INPUT_DECODE_FAILURE = 3116;
public static final int FAILED_TO_PARSE_MALFORMED_LOG_RECORD = 3117;
+ public static final int METADATA_DROP_LIBRARY_IN_USE = 3118;
// Lifecycle management errors
public static final int DUPLICATE_PARTITION_ID = 4000;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index fc356e9..4461ff1 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -312,6 +312,7 @@
3115 = Failed to parse record metadata
3116 = Failed to decode input
3117 = Failed to parse record, malformed log record
+3118 = Library %1$s is being used. It cannot be dropped
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index cf7562f..ca98328 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -534,6 +534,11 @@ public class MetadataNode implements IMetadataNode {
true);
}
+ //Drop libraries, similarly.
+ for (Library lib : getDataverseLibraries(txnId, dataverseName)) {
+ dropLibrary(txnId, lib.getDataverseName(), lib.getName());
+ }
+
List<Dataset> dataverseDatasets;
Dataset ds;
dataverseDatasets = getDataverseDatasets(txnId, dataverseName);