You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2023/12/20 05:35:20 UTC
(asterixdb) branch master updated: [ASTERIXDB-3322][FUN] Adapt external artifacts to DB
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ac782ed17c [ASTERIXDB-3322][FUN] Adapt external artifacts to DB
ac782ed17c is described below
commit ac782ed17c211fe2386d3aa4ba0974eebaad930a
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue Dec 12 08:08:58 2023 -0800
[ASTERIXDB-3322][FUN] Adapt external artifacts to DB
- user model changes: yes
- storage format changes: yes
- interface changes: yes
Details:
Adapt libraries and consumers of libraries to use
the new Database level when appropriate. Add a test
to run some of the library-related tests with this.
Change-Id: I809109b232bc5a5db0c47a52cb98c838ff55ea7e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17988
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
asterixdb/asterix-app/pom.xml | 4 +-
.../asterix/api/http/server/NCUdfApiServlet.java | 6 +-
.../app/external/ExternalLibraryJobUtils.java | 27 +++---
.../asterix/app/translator/QueryTranslator.java | 22 ++---
.../org/apache/asterix/utils/FeedOperations.java | 6 +-
.../test/cloud_storage/CloudPythonTest.java | 96 +++++++++++++++++++
.../test/runtime/ExternalPythonFunctionIT.java | 2 +-
.../library_list_api.5.regexjson | 6 +-
.../resources/runtimets/testsuite_it_python.xml | 1 +
.../asterix/common/api/INamespacePathResolver.java | 2 +
.../asterix/common/library/ILibraryManager.java | 14 +--
.../apache/asterix/common/metadata/Namespace.java | 10 +-
.../common/metadata/NamespacePathResolver.java | 5 +
.../adapter/factory/ExternalAdapterFactory.java | 14 ++-
.../external/library/ExternalLibraryManager.java | 106 ++++++++++++---------
.../ExternalScalarJavaFunctionEvaluator.java | 6 +-
.../PythonLibraryDomainSocketEvaluator.java | 5 +-
.../external/library/PythonLibraryEvaluatorId.java | 16 ++--
.../library/PythonLibraryTCPSocketEvaluator.java | 5 +-
.../AbstractLibraryOperatorDescriptor.java | 12 +--
.../operators/FeedIntakeOperatorDescriptor.java | 16 +++-
.../LibraryDeployAbortOperatorDescriptor.java | 8 +-
.../LibraryDeployCommitOperatorDescriptor.java | 8 +-
.../LibraryDeployPrepareOperatorDescriptor.java | 8 +-
.../LibraryUndeployOperatorDescriptor.java | 8 +-
.../asterix/external/util/ExternalDataUtils.java | 10 +-
.../external/util/ExternalLibraryUtils.java | 10 +-
.../metadata/declared/MetadataProvider.java | 6 +-
.../LibraryTupleTranslator.java | 3 +-
.../asterix/metadata/feeds/FeedMetadataUtil.java | 2 +-
.../functions/ExternalFunctionCompilerUtil.java | 7 +-
.../functions/ExternalScalarFunctionInfo.java | 9 +-
.../asterix/om/functions/ExternalFunctionInfo.java | 16 ++--
.../om/functions/IExternalFunctionInfo.java | 4 +-
34 files changed, 315 insertions(+), 165 deletions(-)
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index ed6f5e3261..f285611975 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -448,7 +448,7 @@
<profile>
<id>asterix-gerrit-asterix-app</id>
<properties>
- <test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/SqlppExecutionColumnTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java</test.excludes>
+ <test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/SqlppExecutionColumnTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java,**/CloudPythonTest.java</test.excludes>
<itest.excludes>**/*.java</itest.excludes>
</properties>
<build>
@@ -482,7 +482,7 @@
<id>asterix-gerrit-asterix-app-sql-execution</id>
<properties>
<test.excludes>**/*.java</test.excludes>
- <itest.includes>**/SqlppExecution*IT.java,**/ExternalPythonFunctionIT.java</itest.includes>
+ <itest.includes>**/SqlppExecution*IT.java,**/ExternalPythonFunctionIT.java,**/CloudPythonTest.java</itest.includes>
<failIfNoTests>false</failIfNoTests>
</properties>
</profile>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index 9698f3b136..74beb46b62 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -181,13 +181,13 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
try {
if (localPath.equals("/") || localPath.equals("")) {
//TODO: nicer way to get this into display form?
- Map<DataverseName, Map<String, String>> dvToLibHashes =
+ Map<Namespace, Map<String, String>> dvToLibHashes =
ExternalLibraryUtils.produceLibraryListing(libraryManager);
List<Map<String, Object>> libraryList = new ArrayList<>();
- for (Map.Entry<DataverseName, Map<String, String>> dvAndLibs : dvToLibHashes.entrySet()) {
+ for (Map.Entry<Namespace, Map<String, String>> dvAndLibs : dvToLibHashes.entrySet()) {
for (Map.Entry<String, String> libsInDv : dvAndLibs.getValue().entrySet()) {
Map<String, Object> libraryEntry = new HashMap<>();
- libraryEntry.put(getDataverseKey(), dvAndLibs.getKey().getCanonicalForm());
+ libraryEntry.put(getDataverseKey(), libraryManager.getNsOrDv(dvAndLibs.getKey()));
libraryEntry.put(NAME_KEY, libsInDv.getKey());
libraryEntry.put(FIELD_HASH, libsInDv.getValue());
libraryList.add(libraryEntry);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
index 6c6691b674..f32b8ef246 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
@@ -28,7 +28,7 @@ import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.operators.LibraryDeployAbortOperatorDescriptor;
import org.apache.asterix.external.operators.LibraryDeployCommitOperatorDescriptor;
@@ -51,30 +51,29 @@ public class ExternalLibraryJobUtils {
}
public static Triple<JobSpecification, JobSpecification, JobSpecification> buildCreateLibraryJobSpec(
- DataverseName dataverseName, String libraryName, ExternalFunctionLanguage language, URI downloadURI,
+ Namespace namespace, String libraryName, ExternalFunctionLanguage language, URI downloadURI,
String authToken, MetadataProvider metadataProvider) {
ICcApplicationContext appCtx = metadataProvider.getApplicationContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = getSplitsAndConstraints(appCtx);
- JobSpecification prepareJobSpec = createLibraryPrepareJobSpec(dataverseName, libraryName, language, downloadURI,
+ JobSpecification prepareJobSpec = createLibraryPrepareJobSpec(namespace, libraryName, language, downloadURI,
authToken, appCtx, splitsAndConstraint);
JobSpecification commitJobSpec =
- createLibraryCommitJobSpec(dataverseName, libraryName, appCtx, splitsAndConstraint);
+ createLibraryCommitJobSpec(namespace, libraryName, appCtx, splitsAndConstraint);
- JobSpecification abortJobSpec =
- createLibraryAbortJobSpec(dataverseName, libraryName, appCtx, splitsAndConstraint);
+ JobSpecification abortJobSpec = createLibraryAbortJobSpec(namespace, libraryName, appCtx, splitsAndConstraint);
return new Triple<>(prepareJobSpec, commitJobSpec, abortJobSpec);
}
- private static JobSpecification createLibraryPrepareJobSpec(DataverseName dataverseName, String libraryName,
+ private static JobSpecification createLibraryPrepareJobSpec(Namespace namespace, String libraryName,
ExternalFunctionLanguage language, URI downloadURI, String authToken, ICcApplicationContext appCtx,
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new LibraryDeployPrepareOperatorDescriptor(jobSpec, dataverseName, libraryName,
+ IOperatorDescriptor opDesc = new LibraryDeployPrepareOperatorDescriptor(jobSpec, namespace, libraryName,
language, downloadURI, authToken);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc,
splitsAndConstraint.second);
@@ -82,32 +81,32 @@ public class ExternalLibraryJobUtils {
return jobSpec;
}
- private static JobSpecification createLibraryCommitJobSpec(DataverseName dataverseName, String libraryName,
+ private static JobSpecification createLibraryCommitJobSpec(Namespace namespace, String libraryName,
ICcApplicationContext appCtx, Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new LibraryDeployCommitOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new LibraryDeployCommitOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc,
splitsAndConstraint.second);
return jobSpec;
}
- private static JobSpecification createLibraryAbortJobSpec(DataverseName dataverseName, String libraryName,
+ private static JobSpecification createLibraryAbortJobSpec(Namespace namespace, String libraryName,
ICcApplicationContext appCtx, Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new LibraryDeployAbortOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new LibraryDeployAbortOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc,
splitsAndConstraint.second);
return jobSpec;
}
- public static JobSpecification buildDropLibraryJobSpec(DataverseName dataverseName, String libraryName,
+ public static JobSpecification buildDropLibraryJobSpec(Namespace namespace, String libraryName,
MetadataProvider metadataProvider) {
ICcApplicationContext appCtx = metadataProvider.getApplicationContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = getSplitsAndConstraints(appCtx);
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new LibraryUndeployOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new LibraryUndeployOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc,
splitsAndConstraint.second);
jobSpec.addRoot(opDesc);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 8071fd8a14..f058c41282 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2220,9 +2220,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
List<Library> libraries =
MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx, databaseName, dataverseName);
for (Library library : libraries) {
- //TODO(DB): library database
- jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(dataverseName, library.getName(),
- metadataProvider));
+ jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(stmtDropDataverse.getNamespace(),
+ library.getName(), metadataProvider));
}
jobsToExecute.add(DataverseUtil.dropDataverseJobSpec(dv, metadataProvider));
@@ -3577,6 +3576,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ Namespace stmtActiveNamespace = getActiveNamespace(cls.getNamespace());
try {
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, databaseName, dataverseName);
if (dv == null) {
@@ -3600,9 +3600,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
// #. prepare to create library artifacts in NC.
- //TODO(DB):
Triple<JobSpecification, JobSpecification, JobSpecification> jobSpecs =
- ExternalLibraryJobUtils.buildCreateLibraryJobSpec(dataverseName, libraryName, language,
+ ExternalLibraryJobUtils.buildCreateLibraryJobSpec(stmtActiveNamespace, libraryName, language,
cls.getLocation(), cls.getAuthToken(), metadataProvider);
JobSpecification prepareJobSpec = jobSpecs.first;
JobSpecification commitJobSpec = jobSpecs.second;
@@ -3645,9 +3644,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
} else if (existingLibrary == null) {
// 'commit' job failed for a new library -> try removing the library
try {
- //TODO(DB):
JobSpecification dropLibraryJobSpec = ExternalLibraryJobUtils
- .buildDropLibraryJobSpec(dataverseName, libraryName, metadataProvider);
+ .buildDropLibraryJobSpec(stmtActiveNamespace, libraryName, metadataProvider);
runJob(hcc, dropLibraryJobSpec, jobFlags);
} catch (Exception e2) {
e.addSuppressed(e2);
@@ -3711,6 +3709,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ Namespace stmtActiveNamespace = getActiveNamespace(stmtDropLibrary.getNamespace());
try {
Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, databaseName, dataverseName);
if (dataverse == null) {
@@ -3741,9 +3740,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
library.getLanguage(), library.getHash(), MetadataUtil.PENDING_DROP_OP));
// #. drop library artifacts in NCs.
- //TODO(DB):
JobSpecification jobSpec =
- ExternalLibraryJobUtils.buildDropLibraryJobSpec(dataverseName, libraryName, metadataProvider);
+ ExternalLibraryJobUtils.buildDropLibraryJobSpec(stmtActiveNamespace, libraryName, metadataProvider);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
@@ -5690,11 +5688,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
List<Dataset> datasets = addDatasetDropJob(metadataProvider, sourceLoc, databaseName, mdTxnCtx, jobsToExecute);
// #. prepare jobs which will drop corresponding libraries
- //TODO(DB): library database
List<Library> libraries = MetadataManager.INSTANCE.getDatabaseLibraries(mdTxnCtx, databaseName);
for (Library library : libraries) {
- jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(library.getDataverseName(),
- library.getName(), metadataProvider));
+ jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(
+ new Namespace(library.getDatabaseName(), library.getDataverseName()), library.getName(),
+ metadataProvider));
}
// #. prepare jobs which will drop the database
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index 85014ab373..93233497dd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -249,9 +249,9 @@ public class FeedOperations {
(FeedIntakeOperatorDescriptor) intakeJob.getOperatorMap().get(new OperatorDescriptorId(0));
FeedIntakeOperatorDescriptor ingestionOp;
if (firstOp.getAdaptorFactory() == null) {
- ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed, firstOp.getAdaptorLibraryDataverse(),
- firstOp.getAdaptorLibraryName(), firstOp.getAdaptorFactoryClassName(),
- firstOp.getAdapterOutputType(), firstOp.getPolicyAccessor(),
+ ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed, firstOp.getAdaptorLibraryDatabase(),
+ firstOp.getAdaptorLibraryDataverse(), firstOp.getAdaptorLibraryName(),
+ firstOp.getAdaptorFactoryClassName(), firstOp.getAdapterOutputType(), firstOp.getPolicyAccessor(),
firstOp.getOutputRecordDescriptors()[0]);
} else {
ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed, firstOp.getAdaptorFactory(),
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
new file mode 100644
index 0000000000..643ed802af
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import static org.apache.asterix.test.runtime.ExternalPythonFunctionIT.setNcEndpoints;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.api.common.LocalCloudUtil;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.Description;
+import org.apache.asterix.testframework.xml.TestCase;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Run tests in cloud deployment environment
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudPythonTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final TestCaseContext tcCtx;
+ private static final String SUITE_TESTS = "testsuite_it_python.xml";
+ private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
+ private static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage.conf";
+ private static final String DELTA_RESULT_PATH = "results_cloud";
+ private static final String EXCLUDED_TESTS = "MP";
+
+ public CloudPythonTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ LocalCloudUtil.startS3CloudEnvironment(true);
+ TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
+ testExecutor.executorId = "cloud";
+ testExecutor.stripSubstring = "//DB:";
+ LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE_NAME);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "CloudPythonTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
+ @Test
+ public void test() throws Exception {
+ List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit();
+ Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())));
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static String getText(Description description) {
+ return description == null ? "" : description.getValue();
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
index d72d494d15..2674f41b99 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
@@ -72,7 +72,7 @@ public class ExternalPythonFunctionIT {
LangExecutionUtil.test(tcCtx);
}
- private static void setNcEndpoints(TestExecutor testExecutor) {
+ public static void setNcEndpoints(TestExecutor testExecutor) {
final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
final String ip = InetAddress.getLoopbackAddress().getHostAddress();
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
index f0cb7cf4dd..de8723176d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
@@ -1,5 +1,5 @@
[{
- "dataverse": "external/lib/test",
+ "dataverse": "external.lib.test",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
},
@@ -9,12 +9,12 @@
"name": "testlib"
},
{
- "dataverse": "externallibtest/foo",
+ "dataverse": "externallibtest.foo",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
},
{
- "dataverse": "externallibtest/foo/bar",
+ "dataverse": "externallibtest.foo.bar",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
}]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index 284c2fd242..c6a7cf5a82 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -30,6 +30,7 @@
</test-case>
<test-case FilePath="external-library">
<compilation-unit name="mysentiment_multipart">
+ <description>MP</description>
<output-dir compare="Text">mysentiment</output-dir>
</compilation-unit>
</test-case>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
index 6f33d6f29a..b5fd3a64f3 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
@@ -27,4 +27,6 @@ public interface INamespacePathResolver {
String resolve(String databaseName, DataverseName dataverseName);
+ boolean usingDatabase();
+
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
index acbce6d4d6..2098060ec2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
@@ -26,7 +26,7 @@ import java.security.MessageDigest;
import java.util.List;
import java.util.function.Function;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -38,22 +38,24 @@ import org.apache.hyracks.ipc.impl.IPCSystem;
public interface ILibraryManager {
- List<Pair<DataverseName, String>> getLibraryListing() throws IOException;
+ List<Pair<Namespace, String>> getLibraryListing() throws IOException;
- String getLibraryHash(DataverseName dataverseName, String libraryName) throws IOException;
+ String getLibraryHash(Namespace namespace, String libraryName) throws IOException;
- ILibrary getLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException;
+ ILibrary getLibrary(Namespace namespace, String libraryName) throws HyracksDataException;
- void closeLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException;
+ void closeLibrary(Namespace namespace, String libraryName) throws HyracksDataException;
// deployment helpers
FileReference getStorageDir();
- FileReference getLibraryDir(DataverseName dataverseName, String libraryName) throws HyracksDataException;
+ FileReference getLibraryDir(Namespace namespace, String libraryName) throws HyracksDataException;
FileReference getDistributionDir();
+ String getNsOrDv(Namespace ns);
+
void dropLibraryPath(FileReference fileRef) throws HyracksDataException;
byte[] serializeLibraryDescriptor(LibraryDescriptor libraryDescriptor) throws HyracksDataException;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
index 5b75c797ca..1a14be4742 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
@@ -22,7 +22,7 @@ package org.apache.asterix.common.metadata;
import java.io.Serializable;
import java.util.Objects;
-public final class Namespace implements Serializable {
+public final class Namespace implements Serializable, Comparable<Namespace> {
private static final long serialVersionUID = 1L;
@@ -61,4 +61,12 @@ public final class Namespace implements Serializable {
return Objects.equals(databaseName, that.databaseName) && Objects.equals(dataverseName, that.dataverseName);
}
+ @Override
+ public int compareTo(Namespace that) {
+ int dbComp = databaseName.compareTo(that.getDatabaseName());
+ if (dbComp == 0) {
+ return dataverseName.compareTo(that.getDataverseName());
+ }
+ return dbComp;
+ }
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
index 910c3ef68f..fa1c0b5837 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
@@ -57,4 +57,9 @@ public class NamespacePathResolver implements INamespacePathResolver {
return StoragePathUtil.prepareDataverseName(dataverseName);
}
}
+
+ @Override
+ public boolean usingDatabase() {
+ return usingDatabase;
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
index 21735436dc..ac34471d63 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
@@ -29,6 +29,7 @@ import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
@@ -44,13 +45,14 @@ import org.apache.hyracks.api.exceptions.IWarningCollector;
public final class ExternalAdapterFactory implements ITypedAdapterFactory {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final DataverseName libraryDataverse;
private final String libraryName;
private final String className;
+ private final String libraryDatabase;
private ARecordType outputType;
@@ -60,7 +62,9 @@ public final class ExternalAdapterFactory implements ITypedAdapterFactory {
private transient ICCServiceContext serviceContext;
- public ExternalAdapterFactory(DataverseName libraryDataverse, String libraryName, String className) {
+ public ExternalAdapterFactory(String libraryDatabase, DataverseName libraryDataverse, String libraryName,
+ String className) {
+ this.libraryDatabase = libraryDatabase;
this.libraryDataverse = libraryDataverse;
this.libraryName = libraryName;
this.className = className;
@@ -85,7 +89,7 @@ public final class ExternalAdapterFactory implements ITypedAdapterFactory {
INCServiceContext serviceCtx = ctx.getJobletContext().getServiceContext();
INcApplicationContext appCtx = (INcApplicationContext) serviceCtx.getApplicationContext();
ILibraryManager libraryManager = appCtx.getLibraryManager();
- ILibrary library = libraryManager.getLibrary(libraryDataverse, libraryName);
+ ILibrary library = libraryManager.getLibrary(new Namespace(libraryDatabase, libraryDataverse), libraryName);
if (ExternalFunctionLanguage.JAVA != library.getLanguage()) {
throw new HyracksDataException("Unexpected library language: " + library.getLanguage());
}
@@ -127,6 +131,10 @@ public final class ExternalAdapterFactory implements ITypedAdapterFactory {
return "external:" + className;
}
+ public String getLibraryDatabase() {
+ return libraryDatabase;
+ }
+
public DataverseName getLibraryDataverse() {
return libraryDataverse;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index 59b1a02cb7..47a17680b9 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -64,6 +64,8 @@ import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.library.LibraryDescriptor;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.MetadataConstants;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
import org.apache.commons.codec.digest.DigestUtils;
@@ -139,7 +141,7 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
private final FileReference distDir;
private final Path trashDirPath;
//TODO(DB): change for database
- private final Map<Pair<DataverseName, String>, ILibrary> libraries = new HashMap<>();
+ private final Map<Pair<Namespace, String>, ILibrary> libraries = new HashMap<>();
private IPCSystem pythonIPC;
private final ExternalFunctionResultRouter router;
private final IIOManager ioManager;
@@ -219,7 +221,7 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
@Override
public void stop(boolean dumpState, OutputStream ouputStream) {
synchronized (this) {
- for (Map.Entry<Pair<DataverseName, String>, ILibrary> p : libraries.entrySet()) {
+ for (Map.Entry<Pair<Namespace, String>, ILibrary> p : libraries.entrySet()) {
ILibrary library = p.getValue();
try {
library.close();
@@ -235,13 +237,13 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
return storageDir;
}
- private FileReference getDataverseDir(DataverseName dataverseName) throws HyracksDataException {
- return getChildFileRef(storageDir, StoragePathUtil.prepareDataverseName(dataverseName));
+ private FileReference getDataverseDir(Namespace namespace) throws HyracksDataException {
+ return getChildFileRef(storageDir, namespacePathResolver.resolve(namespace));
}
@Override
- public FileReference getLibraryDir(DataverseName dataverseName, String libraryName) throws HyracksDataException {
- FileReference dataverseDir = getDataverseDir(dataverseName);
+ public FileReference getLibraryDir(Namespace namespace, String libraryName) throws HyracksDataException {
+ FileReference dataverseDir = getDataverseDir(namespace);
return getChildFileRef(dataverseDir, libraryName);
}
@@ -251,8 +253,8 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
}
@Override
- public List<Pair<DataverseName, String>> getLibraryListing() throws IOException {
- List<Pair<DataverseName, String>> libs = new ArrayList<>();
+ public List<Pair<Namespace, String>> getLibraryListing() throws IOException {
+ List<Pair<Namespace, String>> libs = new ArrayList<>();
Files.walkFileTree(storageDirPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path currPath, BasicFileAttributes attrs) {
@@ -277,30 +279,41 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
//? shouldn't happen
return FileVisitResult.TERMINATE;
}
- //add first part, then all multiparts
- dvParts.add(tokens[0]);
- int currToken = 1;
- for (; currToken < tokens.length && tokens[currToken]
- .codePointAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER; currToken++) {
- dvParts.add(tokens[currToken].substring(1));
- }
- //we should only arrive at foo/^bar/^baz/.../^bat/lib
- //anything else is fishy or empty
- if (currToken != tokens.length - 1) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- String candidateLib = tokens[currToken];
- DataverseName candidateDv;
try {
+ String candidateDb = MetadataConstants.DEFAULT_DATABASE;
+ if (namespacePathResolver.usingDatabase()) {
+ if (tokens.length < 3) {
+ return FileVisitResult.TERMINATE;
+ }
+ libs.add(new Pair<>(new Namespace(candidateDb, DataverseName.create(List.of(tokens[1]))),
+ tokens[3]));
+ return FileVisitResult.SKIP_SUBTREE;
+
+ }
+ //add first part, then look for multiparts
+ dvParts.add(tokens[0]);
+ int currToken = 1;
+ for (; currToken < tokens.length && tokens[currToken]
+ .codePointAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER; currToken++) {
+ dvParts.add(tokens[currToken].substring(1));
+ }
+ //we should only arrive at foo/^bar/^baz/.../^bat/lib
+ //anything else is fishy or empty
+ if (currToken != tokens.length - 1) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ String candidateLib = tokens[currToken];
+ DataverseName candidateDv;
candidateDv = DataverseName.create(dvParts);
+ Namespace candidateNs = new Namespace(candidateDb, candidateDv);
+ FileReference candidateLibPath = findLibraryRevDir(candidateNs, candidateLib);
+ if (candidateLibPath != null) {
+ libs.add(new Pair<>(candidateNs, candidateLib));
+ }
} catch (AsterixException e) {
// shouldn't happen
throw HyracksDataException.create(e);
}
- FileReference candidateLibPath = findLibraryRevDir(candidateDv, candidateLib);
- if (candidateLibPath != null) {
- libs.add(new Pair<>(candidateDv, candidateLib));
- }
return FileVisitResult.SKIP_SUBTREE;
}
});
@@ -308,8 +321,8 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
}
@Override
- public String getLibraryHash(DataverseName dataverseName, String libraryName) throws IOException {
- FileReference revDir = findLibraryRevDir(dataverseName, libraryName);
+ public String getLibraryHash(Namespace namespace, String libraryName) throws IOException {
+ FileReference revDir = findLibraryRevDir(namespace, libraryName);
if (revDir == null) {
throw HyracksDataException
.create(AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION, "Library does not exist"));
@@ -319,26 +332,26 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
}
@Override
- public ILibrary getLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ public ILibrary getLibrary(Namespace namespace, String libraryName) throws HyracksDataException {
+ Pair<Namespace, String> key = getKey(namespace, libraryName);
synchronized (this) {
ILibrary library = libraries.get(key);
if (library == null) {
- library = loadLibrary(dataverseName, libraryName);
+ library = loadLibrary(namespace, libraryName);
libraries.put(key, library);
}
return library;
}
}
- private ILibrary loadLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException {
- FileReference libRevDir = findLibraryRevDir(dataverseName, libraryName);
+ private ILibrary loadLibrary(Namespace namespace, String libraryName) throws HyracksDataException {
+ FileReference libRevDir = findLibraryRevDir(namespace, libraryName);
if (libRevDir == null) {
- throw new HyracksDataException("Cannot find library: " + dataverseName + '.' + libraryName);
+ throw new HyracksDataException("Cannot find library: " + namespace + '.' + libraryName);
}
FileReference libContentsDir = libRevDir.getChild(CONTENTS_DIR_NAME);
if (!libContentsDir.getFile().isDirectory()) {
- throw new HyracksDataException("Cannot find library: " + dataverseName + '.' + libraryName);
+ throw new HyracksDataException("Cannot find library: " + namespace + '.' + libraryName);
}
try {
ExternalFunctionLanguage libLang = getLibraryDescriptor(libRevDir).getLanguage();
@@ -351,7 +364,7 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
throw new HyracksDataException("Invalid language: " + libraryName);
}
} catch (IOException e) {
- LOGGER.error("Failed to initialize library " + dataverseName + '.' + libraryName, e);
+ LOGGER.error("Failed to initialize library " + namespace + '.' + libraryName, e);
throw HyracksDataException.create(e);
}
}
@@ -377,9 +390,8 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
}
- private FileReference findLibraryRevDir(DataverseName dataverseName, String libraryName)
- throws HyracksDataException {
- FileReference libraryBaseDir = getLibraryDir(dataverseName, libraryName);
+ private FileReference findLibraryRevDir(Namespace namespace, String libraryName) throws HyracksDataException {
+ FileReference libraryBaseDir = getLibraryDir(namespace, libraryName);
if (!libraryBaseDir.getFile().isDirectory()) {
return null;
}
@@ -395,8 +407,8 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
}
@Override
- public void closeLibrary(DataverseName dataverseName, String libraryName) throws HyracksDataException {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ public void closeLibrary(Namespace namespace, String libraryName) throws HyracksDataException {
+ Pair<Namespace, String> key = getKey(namespace, libraryName);
ILibrary library;
synchronized (this) {
library = libraries.remove(key);
@@ -410,8 +422,16 @@ public class ExternalLibraryManager implements ILibraryManager, ILifeCycleCompon
public void dumpState(OutputStream os) {
}
- private static Pair<DataverseName, String> getKey(DataverseName dataverseName, String libraryName) {
- return new Pair<>(dataverseName, libraryName);
+ private static Pair<Namespace, String> getKey(Namespace namespace, String libraryName) {
+ return new Pair<>(namespace, libraryName);
+ }
+
+ @Override
+ public String getNsOrDv(Namespace ns) {
+ if (namespacePathResolver.usingDatabase()) {
+ return ns.toString();
+ }
+ return ns.getDataverseName().toString();
}
public Path zipAllLibs() throws IOException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index 4aa5f421a7..d190f5e1dc 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -23,7 +23,7 @@ import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
@@ -49,9 +49,9 @@ class ExternalScalarJavaFunctionEvaluator extends ExternalScalarFunctionEvaluato
IEvaluatorContext context) throws HyracksDataException {
super(finfo, args, argTypes, context);
- DataverseName libraryDataverseName = finfo.getLibraryDataverseName();
+ Namespace libraryNamespace = finfo.getLibraryNamespace();
String libraryName = finfo.getLibraryName();
- JavaLibrary library = (JavaLibrary) libraryManager.getLibrary(libraryDataverseName, libraryName);
+ JavaLibrary library = (JavaLibrary) libraryManager.getLibrary(libraryNamespace, libraryName);
String classname = finfo.getExternalIdentifier().get(0);
try {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
index 056aa9a014..32186338d9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
@@ -62,8 +62,7 @@ public class PythonLibraryDomainSocketEvaluator extends AbstractLibrarySocketEva
public void start() throws IOException, AsterixException {
PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
- PythonLibrary library =
- (PythonLibrary) libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ PythonLibrary library = (PythonLibrary) libMgr.getLibrary(fnId.getLibraryNamespace(), fnId.getLibraryName());
String wd = library.getFile().getAbsolutePath();
MethodHandles.Lookup lookup = MethodHandles.lookup();
SocketAddress sockAddr;
@@ -107,7 +106,7 @@ public class PythonLibraryDomainSocketEvaluator extends AbstractLibrarySocketEva
static PythonLibraryDomainSocketEvaluator getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
IHyracksTaskContext ctx, IWarningCollector warningCollector, SourceLocation sourceLoc)
throws IOException, AsterixException {
- PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryNamespace(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryDomainSocketEvaluator evaluator =
(PythonLibraryDomainSocketEvaluator) ctx.getStateObject(evaluatorId);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
index c2f6f00518..4a6a5e5964 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
@@ -20,18 +20,18 @@ package org.apache.asterix.external.library;
import java.util.Objects;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
final class PythonLibraryEvaluatorId {
- private final DataverseName libraryDataverseName;
+ private final Namespace libraryNamespace;
private final String libraryName;
private final Thread thread;
- PythonLibraryEvaluatorId(DataverseName libraryDataverseName, String libraryName, Thread thread) {
- this.libraryDataverseName = Objects.requireNonNull(libraryDataverseName);
+ PythonLibraryEvaluatorId(Namespace libraryNamespace, String libraryName, Thread thread) {
+ this.libraryNamespace = Objects.requireNonNull(libraryNamespace);
this.libraryName = Objects.requireNonNull(libraryName);
this.thread = Objects.requireNonNull(thread);
}
@@ -43,17 +43,17 @@ final class PythonLibraryEvaluatorId {
if (o == null || getClass() != o.getClass())
return false;
PythonLibraryEvaluatorId that = (PythonLibraryEvaluatorId) o;
- return libraryDataverseName.equals(that.libraryDataverseName) && libraryName.equals(that.libraryName)
+ return libraryNamespace.equals(that.libraryNamespace) && libraryName.equals(that.libraryName)
&& thread.equals(that.thread);
}
@Override
public int hashCode() {
- return Objects.hash(libraryDataverseName, libraryName);
+ return Objects.hash(libraryNamespace, libraryName);
}
- public DataverseName getLibraryDataverseName() {
- return libraryDataverseName;
+ public Namespace getLibraryNamespace() {
+ return libraryNamespace;
}
public String getLibraryName() {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
index 385d738f83..4406cc89eb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
@@ -69,8 +69,7 @@ public class PythonLibraryTCPSocketEvaluator extends AbstractLibrarySocketEvalua
@Override
public void start() throws IOException, AsterixException {
PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
- PythonLibrary library =
- (PythonLibrary) libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ PythonLibrary library = (PythonLibrary) libMgr.getLibrary(fnId.getLibraryNamespace(), fnId.getLibraryName());
String wd = library.getFile().getAbsolutePath();
int port = ipcSys.getSocketAddress().getPort();
List<String> args = new ArrayList<>();
@@ -110,7 +109,7 @@ public class PythonLibraryTCPSocketEvaluator extends AbstractLibrarySocketEvalua
ExternalFunctionResultRouter router, IPCSystem ipcSys, File pythonHome, IHyracksTaskContext ctx,
String sitePkgs, List<String> pythonArgs, Map<String, String> pythonEnv, IWarningCollector warningCollector,
SourceLocation sourceLoc) throws IOException, AsterixException {
- PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(finfo.getLibraryNamespace(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryTCPSocketEvaluator evaluator = (PythonLibraryTCPSocketEvaluator) ctx.getStateObject(evaluatorId);
if (evaluator == null) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
index ec05dfe31a..b25483657f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
@@ -27,7 +27,7 @@ import java.nio.file.StandardCopyOption;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -44,14 +44,14 @@ abstract class AbstractLibraryOperatorDescriptor extends AbstractSingleActivityO
private static final long serialVersionUID = 1L;
- protected final DataverseName dataverseName;
+ protected final Namespace namespace;
protected final String libraryName;
- public AbstractLibraryOperatorDescriptor(IOperatorDescriptorRegistry spec, DataverseName dataverseName,
+ public AbstractLibraryOperatorDescriptor(IOperatorDescriptorRegistry spec, Namespace namespace,
String libraryName) {
super(spec, 0, 0);
- this.dataverseName = dataverseName;
+ this.namespace = namespace;
this.libraryName = libraryName;
}
@@ -77,7 +77,7 @@ abstract class AbstractLibraryOperatorDescriptor extends AbstractSingleActivityO
(INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
ioManager = runtimeCtx.getIoManager();
libraryManager = runtimeCtx.getLibraryManager();
- libraryDir = libraryManager.getLibraryDir(dataverseName, libraryName);
+ libraryDir = libraryManager.getLibraryDir(namespace, libraryName);
try {
execute();
} catch (IOException e) {
@@ -130,7 +130,7 @@ abstract class AbstractLibraryOperatorDescriptor extends AbstractSingleActivityO
}
protected void closeLibrary() throws HyracksDataException {
- libraryManager.closeLibrary(dataverseName, libraryName);
+ libraryManager.closeLibrary(namespace, libraryName);
}
@Override
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index dbc571496c..6a80750a82 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -28,6 +28,7 @@ import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
@@ -63,6 +64,7 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
private final FeedPolicyAccessor policyAccessor;
private final ARecordType adapterOutputType;
+ private String adaptorLibraryDatabase;
/**
* The adaptor factory that is used to create an instance of the feed adaptor
**/
@@ -93,13 +95,14 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
this.outRecDescs[0] = rDesc;
}
- public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed feed, DataverseName adapterLibraryDataverse,
- String adapterLibraryName, String adapterFactoryClassName, ARecordType adapterOutputType,
- FeedPolicyAccessor policyAccessor, RecordDescriptor rDesc) {
+ public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed feed, String databaseName,
+ DataverseName adapterLibraryDataverse, String adapterLibraryName, String adapterFactoryClassName,
+ ARecordType adapterOutputType, FeedPolicyAccessor policyAccessor, RecordDescriptor rDesc) {
super(spec, 0, 1);
this.feedId =
new EntityId(FEED_EXTENSION_NAME, feed.getDatabaseName(), feed.getDataverseName(), feed.getFeedName());
this.adaptorFactoryClassName = adapterFactoryClassName;
+ this.adaptorLibraryDatabase = databaseName;
this.adaptorLibraryDataverse = adapterLibraryDataverse;
this.adaptorLibraryName = adapterLibraryName;
this.adaptorConfiguration = feed.getConfiguration();
@@ -122,7 +125,8 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
INcApplicationContext runtimeCtx =
(INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
- ILibrary lib = libraryManager.getLibrary(adaptorLibraryDataverse, adaptorLibraryName);
+ ILibrary lib = libraryManager.getLibrary(new Namespace(adaptorLibraryDatabase, adaptorLibraryDataverse),
+ adaptorLibraryName);
if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
}
@@ -166,6 +170,10 @@ public class FeedIntakeOperatorDescriptor extends AbstractSingleActivityOperator
return this.policyAccessor;
}
+ public String getAdaptorLibraryDatabase() {
+ return adaptorLibraryDatabase;
+ }
+
public DataverseName getAdaptorLibraryDataverse() {
return adaptorLibraryDataverse;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
index 97eb3e8ea4..4fc8788208 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -36,9 +36,9 @@ public final class LibraryDeployAbortOperatorDescriptor extends AbstractLibraryO
private static final Logger LOGGER = LogManager.getLogger(LibraryDeployAbortOperatorDescriptor.class);
- public LibraryDeployAbortOperatorDescriptor(IOperatorDescriptorRegistry spec, DataverseName dataverseName,
+ public LibraryDeployAbortOperatorDescriptor(IOperatorDescriptorRegistry spec, Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -48,7 +48,7 @@ public final class LibraryDeployAbortOperatorDescriptor extends AbstractLibraryO
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Abort deployment of library {}.{}", dataverseName, libraryName);
+ LOGGER.info("Abort deployment of library {}.{}", namespace, libraryName);
}
FileReference libDir = getLibraryDir();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
index 6cc3df7f7b..c7ca8e46af 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -36,9 +36,9 @@ public final class LibraryDeployCommitOperatorDescriptor extends AbstractLibrary
private static final Logger LOGGER = LogManager.getLogger(LibraryDeployCommitOperatorDescriptor.class);
- public LibraryDeployCommitOperatorDescriptor(IOperatorDescriptorRegistry spec, DataverseName dataverseName,
+ public LibraryDeployCommitOperatorDescriptor(IOperatorDescriptorRegistry spec, Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -48,7 +48,7 @@ public final class LibraryDeployCommitOperatorDescriptor extends AbstractLibrary
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Commit deployment of library {}.{}", dataverseName, libraryName);
+ LOGGER.info("Commit deployment of library {}.{}", namespace, libraryName);
}
// #. rename 'stage' dir into 'rev_1' dir
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
index c12bb58cc7..0c12d4338b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
@@ -33,7 +33,7 @@ import java.security.MessageDigest;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.LibraryDescriptor;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.external.util.ExternalLibraryUtils;
import org.apache.commons.io.FilenameUtils;
@@ -56,9 +56,9 @@ public class LibraryDeployPrepareOperatorDescriptor extends AbstractLibraryOpera
private final URI libLocation;
private final String authToken;
- public LibraryDeployPrepareOperatorDescriptor(IOperatorDescriptorRegistry spec, DataverseName dataverseName,
+ public LibraryDeployPrepareOperatorDescriptor(IOperatorDescriptorRegistry spec, Namespace namespace,
String libraryName, ExternalFunctionLanguage language, URI libLocation, String authToken) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
this.language = language;
this.libLocation = libLocation;
this.authToken = authToken;
@@ -74,7 +74,7 @@ public class LibraryDeployPrepareOperatorDescriptor extends AbstractLibraryOpera
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Prepare deployment of library {}.{}", dataverseName, libraryName);
+ LOGGER.info("Prepare deployment of library {}.{}", namespace, libraryName);
}
// #. create library dir if necessary, clean 'stage' dir
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
index a146ee30e4..04c9c95d8e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -37,9 +37,9 @@ public final class LibraryUndeployOperatorDescriptor extends AbstractLibraryOper
private static final Logger LOGGER = LogManager.getLogger(LibraryUndeployOperatorDescriptor.class);
- public LibraryUndeployOperatorDescriptor(IOperatorDescriptorRegistry spec, DataverseName dataverseName,
+ public LibraryUndeployOperatorDescriptor(IOperatorDescriptorRegistry spec, Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -49,7 +49,7 @@ public final class LibraryUndeployOperatorDescriptor extends AbstractLibraryOper
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Undeploying library {}.{}", dataverseName, libraryName);
+ LOGGER.info("Undeploying library {}.{}", namespace, libraryName);
}
try {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 439e8529d8..189e0d8e78 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.common.metadata.MetadataConstants.DEFAULT_DATABASE;
import static org.apache.asterix.external.util.ExternalDataConstants.DEFINITION_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
@@ -62,6 +63,7 @@ import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
@@ -194,11 +196,11 @@ public class ExternalDataUtils {
}
public static IInputStreamFactory createExternalInputStreamFactory(ILibraryManager libraryManager,
- DataverseName dataverse, String stream) throws HyracksDataException {
+ Namespace namespace, String stream) throws HyracksDataException {
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
- ILibrary lib = libraryManager.getLibrary(dataverse, libraryName);
+ ILibrary lib = libraryManager.getLibrary(namespace, libraryName);
if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
}
@@ -287,7 +289,7 @@ public class ExternalDataUtils {
String libraryName = dataverseAndLibrary[1];
ILibrary lib;
try {
- lib = libraryManager.getLibrary(dataverseName, libraryName);
+ lib = libraryManager.getLibrary(new Namespace(DEFAULT_DATABASE, dataverseName), libraryName);
} catch (HyracksDataException e) {
throw new AsterixException("Cannot load library", e);
}
@@ -310,7 +312,7 @@ public class ExternalDataUtils {
parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR));
ILibrary lib;
try {
- lib = libraryManager.getLibrary(dataverse, library);
+ lib = libraryManager.getLibrary(new Namespace(DEFAULT_DATABASE, dataverse), library);
} catch (HyracksDataException e) {
throw new AsterixException("Cannot load library", e);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
index 9ab07eb03e..57fc90d9b7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.util.bytes.HexPrinter;
@@ -42,11 +42,11 @@ public class ExternalLibraryUtils {
return hashBuilder.toString();
}
- public static Map<DataverseName, Map<String, String>> produceLibraryListing(ILibraryManager libraryManager)
+ public static Map<Namespace, Map<String, String>> produceLibraryListing(ILibraryManager libraryManager)
throws IOException {
- List<Pair<DataverseName, String>> libs = libraryManager.getLibraryListing();
- Map<DataverseName, Map<String, String>> dvToLibHashes = new TreeMap<>();
- for (Pair<DataverseName, String> lib : libs) {
+ List<Pair<Namespace, String>> libs = libraryManager.getLibraryListing();
+ Map<Namespace, Map<String, String>> dvToLibHashes = new TreeMap<>();
+ for (Pair<Namespace, String> lib : libs) {
dvToLibHashes.computeIfAbsent(lib.first, h -> new TreeMap<>()).put(lib.getSecond(),
libraryManager.getLibraryHash(lib.first, lib.second));
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index b310271270..0aa5f460c1 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -578,9 +578,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
break;
case EXTERNAL:
ExternalAdapterFactory extAdapterFactory = (ExternalAdapterFactory) adapterFactory;
- feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed, extAdapterFactory.getLibraryDataverse(),
- extAdapterFactory.getLibraryName(), extAdapterFactory.getClassName(), recordType,
- policyAccessor, factoryOutput.second);
+ feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed, extAdapterFactory.getLibraryDatabase(),
+ extAdapterFactory.getLibraryDataverse(), extAdapterFactory.getLibraryName(),
+ extAdapterFactory.getClassName(), recordType, policyAccessor, factoryOutput.second);
break;
default:
break;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
index 63a252ee29..534a692986 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
@@ -84,12 +84,12 @@ public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> {
String dataverseCanonicalName = library.getDataverseName().getCanonicalForm();
// write the key in the first 2 fields of the tuple
+ tupleBuilder.reset();
if (libraryEntity.databaseNameIndex() >= 0) {
aString.setValue(library.getDatabaseName());
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
}
- tupleBuilder.reset();
aString.setValue(dataverseCanonicalName);
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
@@ -106,6 +106,7 @@ public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> {
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(libraryEntity.databaseNameIndex(), fieldValue);
}
+
// write field 0
fieldValue.reset();
aString.setValue(dataverseCanonicalName);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index ca50fcdae2..4750591870 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -188,7 +188,7 @@ public class FeedMetadataUtil {
if (!ExternalFunctionLanguage.JAVA.name().equals(library.getLanguage())) {
throw new HyracksDataException("Unexpected library language: " + library.getLanguage());
}
- return new ExternalAdapterFactory(libraryDataverse, libraryName, adapterFactoryClassname);
+ return new ExternalAdapterFactory(libraryDatabaseName, libraryDataverse, libraryName, adapterFactoryClassname);
}
@SuppressWarnings("rawtypes")
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 5f42089077..404c4e7dbc 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -26,6 +26,7 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
@@ -81,8 +82,10 @@ public class ExternalFunctionCompilerUtil {
//TODO(DB): review
return new ExternalScalarFunctionInfo(function.getSignature().createFunctionIdentifier(), paramTypes,
- returnType, typeComputer, lang, function.getLibraryDataverseName(), function.getLibraryName(),
- function.getExternalIdentifier(), function.getResources(), deterministic, function.getNullCall());
+ returnType, typeComputer, lang,
+ new Namespace(function.getLibraryDatabaseName(), function.getLibraryDataverseName()),
+ function.getLibraryName(), function.getExternalIdentifier(), function.getResources(), deterministic,
+ function.getNullCall());
}
private static IFunctionInfo getUnnestFunctionInfo(MetadataProvider metadataProvider, Function function) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
index 82f74d9d7b..441d118f65 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
@@ -34,10 +34,9 @@ public class ExternalScalarFunctionInfo extends ExternalFunctionInfo {
private static final long serialVersionUID = 3L;
public ExternalScalarFunctionInfo(FunctionIdentifier fid, List<IAType> parameterTypes, IAType returnType,
- IResultTypeComputer rtc, ExternalFunctionLanguage language, DataverseName libraryDataverseName,
- String libraryName, List<String> externalIdentifier, Map<String, String> resources, boolean deterministic,
- boolean nullCall) {
- super(fid, FunctionKind.SCALAR, parameterTypes, returnType, rtc, language, libraryDataverseName, libraryName,
+ IResultTypeComputer rtc, ExternalFunctionLanguage language, Namespace libraryNamespace, String libraryName,
+ List<String> externalIdentifier, Map<String, String> resources, boolean deterministic, boolean nullCall) {
+ super(fid, FunctionKind.SCALAR, parameterTypes, returnType, rtc, language, libraryNamespace, libraryName,
externalIdentifier, resources, deterministic, nullCall);
}
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index 8a2cefa02a..700c9f2444 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -36,22 +36,22 @@ public class ExternalFunctionInfo extends FunctionInfo implements IExternalFunct
private final List<IAType> parameterTypes;
private final IAType returnType;
private final ExternalFunctionLanguage language;
- private final DataverseName libraryDataverseName;
+ private final Namespace libraryNamespace;
private final String libraryName;
private final List<String> externalIdentifier;
private final Map<String, String> resources;
private final boolean nullCall;
public ExternalFunctionInfo(FunctionIdentifier fid, FunctionKind kind, List<IAType> parameterTypes,
- IAType returnType, IResultTypeComputer rtc, ExternalFunctionLanguage language,
- DataverseName libraryDataverseName, String libraryName, List<String> externalIdentifier,
- Map<String, String> resources, boolean deterministic, boolean nullCall) {
+ IAType returnType, IResultTypeComputer rtc, ExternalFunctionLanguage language, Namespace libraryNamespace,
+ String libraryName, List<String> externalIdentifier, Map<String, String> resources, boolean deterministic,
+ boolean nullCall) {
super(fid, rtc, deterministic);
this.kind = kind;
this.parameterTypes = parameterTypes;
this.returnType = returnType;
this.language = language;
- this.libraryDataverseName = libraryDataverseName;
+ this.libraryNamespace = libraryNamespace;
this.libraryName = libraryName;
this.externalIdentifier = externalIdentifier;
this.resources = resources;
@@ -78,8 +78,8 @@ public class ExternalFunctionInfo extends FunctionInfo implements IExternalFunct
return language;
}
- public DataverseName getLibraryDataverseName() {
- return libraryDataverseName;
+ public Namespace getLibraryNamespace() {
+ return libraryNamespace;
}
@Override
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
index 9eb9875655..e2662e662a 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -40,7 +40,7 @@ public interface IExternalFunctionInfo extends IFunctionInfo {
ExternalFunctionLanguage getLanguage();
- DataverseName getLibraryDataverseName();
+ Namespace getLibraryNamespace();
String getLibraryName();