You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2020/05/25 11:42:53 UTC
[asterixdb] branch master updated (ed6bf2a -> 9627b2a)
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from ed6bf2a [ASTERIXDB-2732][FUN] Support missing and union type in get_object_field_value
new 1a28c65 [ASTERIXDB-2736][EXT] Ensure retrieving all objects if an S3 bucket has more than 1000 objects
new 0f407d9 [ASTERIXDB-2725][COMP] Check if dataverse exists before continuing the DDL execution
new 9627b2a Merge commit '0f407d9224' from 'gerrit/mad-hatter'
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/app/translator/QueryTranslator.java | 33 +++++++++++++++++--
.../aws/AwsS3ExternalDatasetTest.java | 18 +++++++++++
.../dataset-and-index-same-dataverse.000.ddl.sqlpp | 32 +++++++++++++++++++
...ataset-and-index-same-dataverse.001.query.sqlpp | 24 ++++++++++++++
...dataset-and-index-same-dataverse.999.ddl.sqlpp} | 0
.../invalid-dataverse.001.ddl.sqlpp | 21 ++++++++++++
.../invalid-dataverse.002.ddl.sqlpp | 26 +++++++++++++++
.../invalid-dataverse.003.ddl.sqlpp | 28 ++++++++++++++++
.../invalid-dataverse.004.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.005.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.006.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.007.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.008.ddl.sqlpp | 31 ++++++++++++++++++
.../invalid-dataverse.099.ddl.sqlpp | 21 ++++++++++++
.../over-1000-objects.000.ddl.sqlpp | 37 ++++++++++++++++++++++
.../over-1000-objects.001.query.sqlpp | 22 +++++++++++++
.../over-1000-objects.099.ddl.sqlpp} | 0
.../dataset-and-index-same-dataverse.001.adm | 1 +
.../s3/over-1000-objects/over-1000-objects.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 5 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 20 +++++++++++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 32 ++++++++++++++++---
22 files changed, 459 insertions(+), 9 deletions(-)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.000.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.query.sqlpp
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{string/join/001/join.003.ddl.sqlpp => ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.999.ddl.sqlpp} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.001.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.002.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.003.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.004.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.005.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.006.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.007.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.008.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.099.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.000.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.query.sqlpp
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{warnings/warnings-limit/warnings-limit.09.ddl.sqlpp => external-dataset/aws/s3/over-1000-objects/over-1000-objects.099.ddl.sqlpp} (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.adm
[asterixdb] 01/03: [ASTERIXDB-2736][EXT] Ensure retrieving all
objects if an S3 bucket has more than 1000 objects
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1a28c658810ded5d50ff88f42ef88ec8f82926e1
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Fri May 15 12:50:07 2020 +0300
[ASTERIXDB-2736][EXT] Ensure retrieving all objects if an S3 bucket has more than 1000 objects
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fixed the AWS reader to handle reading more than 1000
objects coming in a single request.
- Added a test case for the above mentioned item.
Change-Id: Ic7891aa86852e07dfad9ce41de908b34f86bdb42
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6344
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../aws/AwsS3ExternalDatasetTest.java | 18 +++++++++++
.../over-1000-objects.000.ddl.sqlpp | 37 ++++++++++++++++++++++
.../over-1000-objects.001.query.sqlpp | 22 +++++++++++++
.../over-1000-objects.099.ddl.sqlpp | 20 ++++++++++++
.../s3/over-1000-objects/over-1000-objects.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 5 +++
.../record/reader/aws/AwsS3InputStreamFactory.java | 32 ++++++++++++++++---
7 files changed, 130 insertions(+), 5 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 55c78e3..23bf3bd 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -103,6 +103,10 @@ public class AwsS3ExternalDatasetTest {
private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
+ // This is used for a test to generate over 1000 number of files
+ private static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
+ private static final int OVER_1000_OBJECTS_COUNT = 2999;
+
private static final Set<String> fileNames = new HashSet<>();
private static final CreateBucketRequest.Builder CREATE_BUCKET_BUILDER = CreateBucketRequest.builder();
private static final DeleteBucketRequest.Builder DELETE_BUCKET_BUILDER = DeleteBucketRequest.builder();
@@ -213,6 +217,10 @@ public class AwsS3ExternalDatasetTest {
LOGGER.info("Adding TSV files to the bucket");
loadTsvFiles();
LOGGER.info("TSV Files added successfully");
+
+ LOGGER.info("Loading " + OVER_1000_OBJECTS_COUNT + " into " + OVER_1000_OBJECTS_PATH);
+ loadLargeNumberOfFiles();
+ LOGGER.info("Added " + OVER_1000_OBJECTS_COUNT + " files into " + OVER_1000_OBJECTS_PATH + " successfully");
}
/**
@@ -381,6 +389,16 @@ public class AwsS3ExternalDatasetTest {
}
}
+ /**
+ * Generates over 1000 objects and upload them to S3 mock server, 1 record per object
+ */
+ private static void loadLargeNumberOfFiles() {
+ for (int i = 0; i < OVER_1000_OBJECTS_COUNT; i++) {
+ RequestBody body = RequestBody.fromString("{\"id\":" + i + "}");
+ client.putObject(builder.key(OVER_1000_OBJECTS_PATH + "/" + i + ".json").build(), body);
+ }
+ }
+
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.000.ddl.sqlpp
new file mode 100644
index 0000000..a26caeb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="over-1000-objects"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.query.sqlpp
new file mode 100644
index 0000000..affdb87
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/over-1000-objects/over-1000-objects.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.adm
new file mode 100644
index 0000000..b610b1d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/over-1000-objects/over-1000-objects.001.adm
@@ -0,0 +1 @@
+{ "count": 2999 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 551d777..5aa1326 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -67,5 +67,10 @@
<output-dir compare="Text">aws/s3/empty-string-definition</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/over-1000-objects">
+ <output-dir compare="Text">aws/s3/over-1000-objects</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 58a77b1..9158a57 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -42,8 +42,8 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
-import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
-import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AwsS3InputStreamFactory implements IInputStreamFactory {
@@ -86,13 +86,35 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
S3Client s3Client = buildAwsS3Client(configuration);
// Get all objects in a bucket and extract the paths to files
- ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder().bucket(container);
+ ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container);
String path = configuration.get(AwsS3Constants.DEFINITION_FIELD_NAME);
if (path != null) {
listObjectsBuilder.prefix(path + (!path.isEmpty() && !path.endsWith("/") ? "/" : ""));
}
- ListObjectsResponse listObjectsResponse = s3Client.listObjects(listObjectsBuilder.build());
- List<S3Object> s3Objects = listObjectsResponse.contents();
+
+ ListObjectsV2Response listObjectsResponse;
+ List<S3Object> s3Objects = new ArrayList<>();
+ boolean done = false;
+ String newMarker = null;
+
+ while (!done) {
+ // List the objects from the start, or from the last marker in case of truncated result
+ if (newMarker == null) {
+ listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.build());
+ } else {
+ listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
+ }
+
+ // Collect all the provided objects
+ s3Objects.addAll(listObjectsResponse.contents());
+
+ // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
+ if (!listObjectsResponse.isTruncated()) {
+ done = true;
+ } else {
+ newMarker = listObjectsResponse.nextContinuationToken();
+ }
+ }
// Exclude the directories and get the files only
String fileFormat = configuration.get(ExternalDataConstants.KEY_FORMAT);
[asterixdb] 03/03: Merge commit '0f407d9224' from
'gerrit/mad-hatter'
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9627b2a371aff80ed87280617d6ce1ce79ce4026
Merge: ed6bf2a 0f407d9
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Thu May 21 14:36:23 2020 +0300
Merge commit '0f407d9224' from 'gerrit/mad-hatter'
Change-Id: I89eaf1bace87130d8d35ec0ce8e0c263ac87b59a
.../asterix/app/translator/QueryTranslator.java | 33 +++++++++++++++++--
.../aws/AwsS3ExternalDatasetTest.java | 18 +++++++++++
.../dataset-and-index-same-dataverse.000.ddl.sqlpp | 32 +++++++++++++++++++
...ataset-and-index-same-dataverse.001.query.sqlpp | 24 ++++++++++++++
.../dataset-and-index-same-dataverse.999.ddl.sqlpp | 21 ++++++++++++
.../invalid-dataverse.001.ddl.sqlpp | 21 ++++++++++++
.../invalid-dataverse.002.ddl.sqlpp | 26 +++++++++++++++
.../invalid-dataverse.003.ddl.sqlpp | 28 ++++++++++++++++
.../invalid-dataverse.004.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.005.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.006.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.007.ddl.sqlpp | 29 +++++++++++++++++
.../invalid-dataverse.008.ddl.sqlpp | 31 ++++++++++++++++++
.../invalid-dataverse.099.ddl.sqlpp | 21 ++++++++++++
.../over-1000-objects.000.ddl.sqlpp | 37 ++++++++++++++++++++++
.../over-1000-objects.001.query.sqlpp | 22 +++++++++++++
.../over-1000-objects.099.ddl.sqlpp | 20 ++++++++++++
.../dataset-and-index-same-dataverse.001.adm | 1 +
.../s3/over-1000-objects/over-1000-objects.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 5 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 20 +++++++++++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 32 ++++++++++++++++---
22 files changed, 500 insertions(+), 9 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 9fdc81a,0f540ff..d40957f
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -669,9 -624,20 +671,15 @@@ public class QueryTranslator extends Ab
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- createDatasetBegin(metadataProvider, dataverseName, dataverseName + "." + datasetName, itemTypeDataverseName,
- itemTypeAnonymous, itemTypeFullyQualifiedName, metaItemTypeDataverseName,
- metaItemTypeFullyQualifiedName, metaItemTypeAnonymous, nodegroupName, compactionPolicy,
- defaultCompactionPolicy, dd);
Dataset dataset = null;
try {
- IDatasetDetails datasetDetails;
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
+ IDatasetDetails datasetDetails = null;
Dataset ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds != null) {
if (dd.getIfNotExists()) {
@@@ -928,10 -906,20 +936,18 @@@
List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- String datasetFullyQualifiedName = dataverseName + "." + datasetName;
boolean isSecondaryPrimary = stmtCreateIndex.getFieldExprs().isEmpty();
+ Dataset ds;
+ Index index;
- MetadataLockUtil.createIndexBegin(lockManager, metadataProvider.getLocks(), dataverseName,
- datasetFullyQualifiedName);
+ lockUtil.createIndexBegin(lockManager, metadataProvider.getLocks(), dataverseName, datasetName);
try {
- Dataset ds = metadataProvider.findDataset(dataverseName, datasetName);
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
+ ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds == null) {
throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName,
dataverseName);
@@@ -1867,8 -1849,15 +1888,14 @@@
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), dataverseName,
- dataverseName + "." + typeName);
+ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), dataverseName, typeName);
try {
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, typeName);
if (dt == null) {
if (!stmtTypeDrop.getIfExists()) {
[asterixdb] 02/03: [ASTERIXDB-2725][COMP] Check if dataverse exists
before continuing the DDL execution
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0f407d9224c92a2a5d571dcd53873da9c287021f
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Fri May 15 14:10:53 2020 +0300
[ASTERIXDB-2725][COMP] Check if dataverse exists before continuing the DDL execution
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added dataverse check at multiple places before proceeding
with the execution of the DDL.
- Added a test cases for each newly added check.
Change-Id: I8d54b0b10741d65df91cbb4b7596d363f7ddbd97
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6203
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../asterix/app/translator/QueryTranslator.java | 25 +++++++++++++++++
.../dataset-and-index-same-dataverse.000.ddl.sqlpp | 32 ++++++++++++++++++++++
...ataset-and-index-same-dataverse.001.query.sqlpp | 24 ++++++++++++++++
.../dataset-and-index-same-dataverse.999.ddl.sqlpp | 21 ++++++++++++++
.../invalid-dataverse.001.ddl.sqlpp | 21 ++++++++++++++
.../invalid-dataverse.002.ddl.sqlpp | 26 ++++++++++++++++++
.../invalid-dataverse.003.ddl.sqlpp | 28 +++++++++++++++++++
.../invalid-dataverse.004.ddl.sqlpp | 29 ++++++++++++++++++++
.../invalid-dataverse.005.ddl.sqlpp | 29 ++++++++++++++++++++
.../invalid-dataverse.006.ddl.sqlpp | 29 ++++++++++++++++++++
.../invalid-dataverse.007.ddl.sqlpp | 29 ++++++++++++++++++++
.../invalid-dataverse.008.ddl.sqlpp | 31 +++++++++++++++++++++
.../invalid-dataverse.099.ddl.sqlpp | 21 ++++++++++++++
.../dataset-and-index-same-dataverse.001.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 20 +++++++++++++-
15 files changed, 365 insertions(+), 1 deletion(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 93b36c5..0f540ff 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.translator;
+import static org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_DATAVERSE;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
@@ -629,6 +631,12 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
defaultCompactionPolicy, dd);
Dataset dataset = null;
try {
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
IDatasetDetails datasetDetails = null;
Dataset ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds != null) {
@@ -905,6 +913,12 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
MetadataLockUtil.createIndexBegin(lockManager, metadataProvider.getLocks(), dataverseName,
datasetFullyQualifiedName);
try {
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds == null) {
throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName,
@@ -1558,6 +1572,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
List<JobSpecification> jobsToExecute = new ArrayList<>();
try {
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx.getValue(), dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
Dataset ds = metadataProvider.findDataset(dataverseName, datasetName);
if (ds == null) {
if (ifExists) {
@@ -1833,6 +1852,12 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), dataverseName,
dataverseName + "." + typeName);
try {
+ // Check if the dataverse exists
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName);
+ }
+
Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, typeName);
if (dt == null) {
if (!stmtTypeDrop.getIfExists()) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.000.ddl.sqlpp
new file mode 100644
index 0000000..0e61805
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.000.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test.myType if exists;
+create type test.myType as open { id: uuid, f1: int };
+
+drop dataset test.myDataset if exists;
+create dataset test.myDataset(test.myType) primary key id autogenerated;
+
+use Default;
+create index myIndex on test.myDataset(f1);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.query.sqlpp
new file mode 100644
index 0000000..2cf6a77
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select m.DataverseName
+from Metadata.`Index` m
+where m.DatasetName = "myDataset"
+and m.IndexName = "myIndex";
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.999.ddl.sqlpp
new file mode 100644
index 0000000..5c1e653
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.999.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.001.ddl.sqlpp
new file mode 100644
index 0000000..0fb09b8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.001.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop type fakeDataverse.myType if exists;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.002.ddl.sqlpp
new file mode 100644
index 0000000..c6715e4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.002.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type fakeDataverse.myType as open { id: uuid, f1: int };
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.003.ddl.sqlpp
new file mode 100644
index 0000000..7d482dd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.003.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset fakeDataverse.myDataset if exists;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.004.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.004.ddl.sqlpp
new file mode 100644
index 0000000..2e33fc3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.004.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset realDataverse.myDataset if exists;
+create dataset fakeDataverse.myDataset(fakeDataverse.myType) primary key id autogenerated;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.005.ddl.sqlpp
new file mode 100644
index 0000000..9723237
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.005.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset realDataverse.myDataset if exists;
+create dataset realDataverse.myDataset(fakeDataverse.myType) primary key id autogenerated;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.006.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.006.ddl.sqlpp
new file mode 100644
index 0000000..9c52c7b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.006.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset realDataverse.myDataset if exists;
+create dataset fakeDataverse.myDataset(realDataverse.myType) primary key id autogenerated;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.007.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.007.ddl.sqlpp
new file mode 100644
index 0000000..56f08fd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.007.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset realDataverse.myDataset if exists;
+create dataset realDataverse.myDataset(realDataverse.myType) primary key id autogenerated;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.008.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.008.ddl.sqlpp
new file mode 100644
index 0000000..9375963
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.008.ddl.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+create dataverse realDataverse;
+use realDataverse;
+
+drop type realDataverse.myType if exists;
+create type realDataverse.myType as open { id: uuid, f1: int };
+
+drop dataset realDataverse.myDataset if exists;
+create dataset realDataverse.myDataset(realDataverse.myType) primary key id autogenerated;
+
+create index myIndex on fakeDataverse.myDataset(id);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.099.ddl.sqlpp
new file mode 100644
index 0000000..5440627
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/invalid-dataverse/invalid-dataverse.099.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse realDataverse if exists;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.adm
new file mode 100644
index 0000000..057d80a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/dataset-and-index-same-dataverse/dataset-and-index-same-dataverse.001.adm
@@ -0,0 +1 @@
+{ "DataverseName": "test" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 3bed78a..f127fbd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3848,7 +3848,7 @@
</test-case>
-->
</test-group>
- <test-group name="ddl/create-index">
+ <test-group name="ddl">
<test-case FilePath="ddl/create-index">
<compilation-unit name="create-index-1">
<output-dir compare="Text">create-index-1</output-dir>
@@ -3896,6 +3896,24 @@
<expected-error>Cannot drop index "ds". Drop dataset "ds" to remove this index</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="invalid-dataverse">
+ <output-dir compare="Text">invalid-dataverse</output-dir>
+ <source-location>false</source-location>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 20, at column 1)</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 25, at column 1)</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 27, at column 1)</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 28, at column 1)</expected-error>
+ <expected-error>Cannot find datatype with name fakeDataverse.myType</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 28, at column 1)</expected-error>
+ <expected-error>Cannot find dataverse with name fakeDataverse (in line 30, at column 1)</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="dataset-and-index-same-dataverse">
+ <output-dir compare="Text">dataset-and-index-same-dataverse</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="dml">
<test-case FilePath="dml">