You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/05/09 14:41:45 UTC
[asterixdb] branch master updated (2364a21 -> 5cc254a)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 2364a21 Merge commit 'bc0f7e0' from branch 'gerrit/mad-hatter'
new 74cecd9 [NO ISSUE] Minor cleanup in external datasets
new 3e2623c [NO ISSUE][COMP] CREATE DATASET with inline type definition
new 5cc254a Merge commit '3e2623cf64827e891ecd8c6dea13d03f5599c6a5' into HEAD
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/app/translator/QueryTranslator.java | 143 +++++++++++++++++----
.../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++++++++++
.../create-dataset-inline-type-1.2.query.sqlpp} | 3 +-
.../create-dataset-inline-type-1.3.ddl.sqlpp} | 19 ++-
.../create-dataset-inline-type-1.4.query.sqlpp} | 13 +-
.../create-dataset-inline-type-1.2.adm | 21 +++
.../create-dataset-inline-type-1.4.adm} | 0
.../results/feeds/feeds_01/feeds_01.1.adm | 2 +-
.../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../adapter/factory/GenericAdapterFactory.java | 20 ++-
.../util/ExternalDataCompatibilityUtils.java | 7 +-
.../external/util/ExternalDataConstants.java | 10 --
asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +-
.../asterix/lang/common/statement/DatasetDecl.java | 32 ++---
.../lang/common/visitor/FormatPrintVisitor.java | 13 +-
.../lang/common/visitor/QueryPrintVisitor.java | 15 ++-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 136 ++++++++++++++++----
.../apache/asterix/metadata/utils/DatasetUtil.java | 20 ++-
.../asterix/metadata/utils/MetadataLockUtil.java | 6 +-
20 files changed, 485 insertions(+), 141 deletions(-)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp
copy asterixdb/asterix-app/src/test/resources/runtimets/{queries/dml/insert-duplicated-keys-from-query/insert-duplicated-keys-from-query.3.ddl.aql => queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp} (97%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{custord/join_q_05/join_q_05.3.query.sqlpp => ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp} (71%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{misc/ifthenelse_01/ifthenelse_01.3.query.sqlpp => ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp} (89%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm
copy asterixdb/{asterix-server/src/test/resources/integrationts/NcLifecycle/results/networking/reuse_data_port/reuse_data_port.8.adm => asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm} (100%)
[asterixdb] 03/03: Merge commit
'3e2623cf64827e891ecd8c6dea13d03f5599c6a5' into HEAD
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5cc254a2014e601f1aa65e0a4fdf98d4ef538318
Merge: 2364a21 3e2623c
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri May 8 17:12:47 2020 -0700
Merge commit '3e2623cf64827e891ecd8c6dea13d03f5599c6a5' into HEAD
Change-Id: I89091d8bf8575c73d26f497d5fbffe780adbedf9
.../asterix/app/translator/QueryTranslator.java | 143 +++++++++++++++++----
.../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++++++++++
.../create-dataset-inline-type-1.2.query.sqlpp | 22 ++++
.../create-dataset-inline-type-1.3.ddl.sqlpp | 30 +++++
.../create-dataset-inline-type-1.4.query.sqlpp | 26 ++++
.../create-dataset-inline-type-1.2.adm | 21 +++
.../create-dataset-inline-type-1.4.adm | 1 +
.../results/feeds/feeds_01/feeds_01.1.adm | 2 +-
.../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../adapter/factory/GenericAdapterFactory.java | 20 ++-
.../util/ExternalDataCompatibilityUtils.java | 7 +-
.../external/util/ExternalDataConstants.java | 10 --
asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +-
.../asterix/lang/common/statement/DatasetDecl.java | 32 ++---
.../lang/common/visitor/FormatPrintVisitor.java | 13 +-
.../lang/common/visitor/QueryPrintVisitor.java | 15 ++-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 136 ++++++++++++++++----
.../apache/asterix/metadata/utils/DatasetUtil.java | 20 ++-
.../asterix/metadata/utils/MetadataLockUtil.java | 6 +-
20 files changed, 548 insertions(+), 122 deletions(-)
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index eefb7f3,a440ae4..cf14555
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -99,9 -91,10 +99,11 @@@ import org.apache.asterix.lang.common.b
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
+ import org.apache.asterix.lang.common.expression.TypeExpression;
+ import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
import org.apache.asterix.lang.common.statement.CreateFeedStatement;
@@@ -581,44 -555,55 +583,74 @@@ public class QueryTranslator extends Ab
}
public void handleCreateDatasetStatement(MetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc, IRequestParameters requestParameters) throws CompilationException, Exception {
- MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS);
+ IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception {
DatasetDecl dd = (DatasetDecl) stmt;
- SourceLocation sourceLoc = dd.getSourceLocation();
- String dataverseName = getActiveDataverse(dd.getDataverse());
+ DataverseName dataverseName = getActiveDataverseName(dd.getDataverse());
String datasetName = dd.getName().getValue();
- DataverseName itemTypeDataverseName = getActiveDataverseName(dd.getItemTypeDataverse());
- String itemTypeName = dd.getItemTypeName().getValue();
- DatasetType dsType = dd.getDatasetType();
+ TypeExpression itemTypeExpr = dd.getItemType();
- String itemTypeDataverseName = null, itemTypeName = null, itemTypeFullyQualifiedName = null;
++ DataverseName itemTypeDataverseName = null;
++ String itemTypeName = null;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression itemTypeRefExpr = (TypeReferenceExpression) itemTypeExpr;
- Identifier itemTypeDataverseIdent = itemTypeRefExpr.getIdent().first;
- itemTypeDataverseName = itemTypeDataverseIdent != null && itemTypeDataverseIdent.getValue() != null
- ? itemTypeDataverseIdent.getValue() : dataverseName;
++ Pair<DataverseName, Identifier> itemTypeIdent = itemTypeRefExpr.getIdent();
++ itemTypeDataverseName = itemTypeIdent.first != null ? itemTypeIdent.first : dataverseName;
+ itemTypeName = itemTypeRefExpr.getIdent().second.getValue();
- itemTypeFullyQualifiedName = itemTypeDataverseName + '.' + itemTypeName;
+ break;
+ case RECORD:
+ break;
+ default:
- throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
++ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(),
+ String.valueOf(itemTypeExpr.getTypeKind()));
+ }
+
+ TypeExpression metaItemTypeExpr = dd.getMetaItemType();
- String metaItemTypeDataverseName = null, metaItemTypeName = null, metaItemTypeFullyQualifiedName = null;
+ DataverseName metaItemTypeDataverseName = null;
+ String metaItemTypeName = null;
- Identifier metaItemTypeId = dd.getMetaItemTypeName();
- if (metaItemTypeId != null) {
- metaItemTypeName = metaItemTypeId.getValue();
- metaItemTypeDataverseName = getActiveDataverseName(dd.getMetaItemTypeDataverse());
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression metaItemTypeRefExpr = (TypeReferenceExpression) metaItemTypeExpr;
- Identifier metaItemTypeDataverseIdent = metaItemTypeRefExpr.getIdent().first;
++ Pair<DataverseName, Identifier> metaItemTypeIdent = metaItemTypeRefExpr.getIdent();
+ metaItemTypeDataverseName =
- metaItemTypeDataverseIdent != null && metaItemTypeDataverseIdent.getValue() != null
- ? metaItemTypeDataverseIdent.getValue() : dataverseName;
++ metaItemTypeIdent.first != null ? metaItemTypeIdent.first : dataverseName;
+ metaItemTypeName = metaItemTypeRefExpr.getIdent().second.getValue();
- metaItemTypeFullyQualifiedName = metaItemTypeDataverseName + '.' + metaItemTypeName;
+ break;
+ case RECORD:
+ break;
+ default:
- throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
++ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(),
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
}
+
Identifier ngNameId = dd.getNodegroupName();
String nodegroupName = ngNameId == null ? null : ngNameId.getValue();
String compactionPolicy = dd.getCompactionPolicy();
+ boolean defaultCompactionPolicy = compactionPolicy == null;
+
+ lockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, datasetName,
+ itemTypeDataverseName, itemTypeName, metaItemTypeDataverseName, metaItemTypeName, nodegroupName,
+ compactionPolicy, defaultCompactionPolicy, dd.getDatasetDetailsDecl());
+ try {
+ doCreateDatasetStatement(metadataProvider, dd, dataverseName, datasetName, itemTypeDataverseName,
- itemTypeName, metaItemTypeDataverseName, metaItemTypeName, hcc, requestParameters);
++ itemTypeExpr, itemTypeName, metaItemTypeExpr, metaItemTypeDataverseName, metaItemTypeName, hcc,
++ requestParameters);
+ } finally {
+ metadataProvider.getLocks().unlock();
+ }
+ }
+
+ protected void doCreateDatasetStatement(MetadataProvider metadataProvider, DatasetDecl dd,
- DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName, String itemTypeName,
++ DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName,
++ TypeExpression itemTypeExpr, String itemTypeName, TypeExpression metaItemTypeExpr,
+ DataverseName metaItemTypeDataverseName, String metaItemTypeName, IHyracksClientConnection hcc,
+ IRequestParameters requestParameters) throws Exception {
+ MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS);
+ SourceLocation sourceLoc = dd.getSourceLocation();
+ DatasetType dsType = dd.getDatasetType();
+ Identifier ngNameId = dd.getNodegroupName();
+ String compactionPolicy = dd.getCompactionPolicy();
Map<String, String> compactionPolicyProperties = dd.getCompactionPolicyProperties();
String compressionScheme = metadataProvider.getCompressionManager()
.getDdlOrDefaultCompressionScheme(dd.getDatasetCompressionScheme());
@@@ -637,10 -627,23 +669,23 @@@
throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName);
}
}
- Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- itemTypeDataverseName, itemTypeName);
- if (dt == null) {
- throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeName);
+ IAType itemType;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ itemType = metadataProvider.findType(itemTypeDataverseName, itemTypeName);
+ break;
+ case RECORD:
+ itemTypeDataverseName = dataverseName;
+ itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false);
- MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
- itemTypeDataverseName + "." + itemTypeName);
++ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
++ itemTypeName);
+ itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(itemTypeDataverseName, itemTypeName, itemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(itemTypeExpr.getTypeKind()));
}
String ngName = ngNameId != null ? ngNameId.getValue()
: configureNodegroupForDataset(appCtx, dd.getHints(), dataverseName, datasetName, metadataProvider,
@@@ -661,12 -663,29 +705,29 @@@
}
IAType metaItemType = null;
- if (metaItemTypeDataverseName != null && metaItemTypeName != null) {
- metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
- }
- if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.OBJECT) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Dataset meta type has to be a record type.");
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
+ if (metaItemType.getTypeTag() != ATypeTag.OBJECT) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Dataset meta type has to be a record type.");
+ }
+ break;
+ case RECORD:
+ metaItemTypeDataverseName = dataverseName;
+ metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true);
- MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
- metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName);
++ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
++ metaItemTypeDataverseName, metaItemTypeName);
+ metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName,
+ metaItemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
}
ARecordType metaRecType = (ARecordType) metaItemType;
@@@ -1290,19 -1313,25 +1348,27 @@@
}
}
- private IAType translateType(String dataverseName, String typeName, TypeExpression typeDef,
++ private IAType translateType(DataverseName dataverseName, String typeName, TypeExpression typeDef,
+ MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
- Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, typeDef, typeName, dataverseName);
++ Map<TypeSignature, IAType> typeMap =
++ TypeTranslator.computeTypes(dataverseName, typeName, typeDef, dataverseName, mdTxnCtx);
+ TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
+ return typeMap.get(typeSignature);
+ }
+
protected void handleDataverseDropStatement(MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception {
- DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt;
- SourceLocation sourceLoc = stmtDelete.getSourceLocation();
- String dataverseName = stmtDelete.getDataverseName().getValue();
- if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)) {
+ DataverseDropStatement stmtDropDataverse = (DataverseDropStatement) stmt;
+ SourceLocation sourceLoc = stmtDropDataverse.getSourceLocation();
+ DataverseName dataverseName = stmtDropDataverse.getDataverseName();
+ if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)
+ || dataverseName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME + " dataverse can't be dropped");
+ dataverseName + " dataverse can't be dropped");
}
- lockManager.acquireDataverseWriteLock(metadataProvider.getLocks(), dataverseName);
+ lockUtil.dropDataverseBegin(lockManager, metadataProvider.getLocks(), dataverseName);
try {
- doDropDataverse(stmtDelete, sourceLoc, metadataProvider, hcc);
+ doDropDataverse(stmtDropDataverse, metadataProvider, hcc, requestParameters);
} finally {
metadataProvider.getLocks().unlock();
ExternalDatasetsRegistry.INSTANCE.releaseAcquiredLocks(metadataProvider);
@@@ -1515,8 -1529,41 +1581,39 @@@
}
}
validateDatasetState(metadataProvider, ds, sourceLoc);
+
+ // prepare to drop item and meta types if they were created as inline types
- String itemTypeDataverseName = ds.getItemTypeDataverseName();
++ DataverseName itemTypeDataverseName = ds.getItemTypeDataverseName();
+ String itemTypeName = ds.getItemTypeName();
+ boolean isInlineItemType = DatasetUtil.isInlineTypeName(ds, itemTypeDataverseName, itemTypeName);
+ if (isInlineItemType) {
- MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
- itemTypeDataverseName + '.' + itemTypeName);
++ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, itemTypeName);
+ }
- String metaTypeDataverseName = ds.getMetaItemTypeDataverseName();
++ DataverseName metaTypeDataverseName = ds.getMetaItemTypeDataverseName();
+ String metaTypeName = ds.getMetaItemTypeName();
+ boolean isInlineMetaType =
+ metaTypeName != null && DatasetUtil.isInlineTypeName(ds, metaTypeDataverseName, metaTypeName);
+ if (isInlineMetaType) {
- MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName,
- metaTypeDataverseName + '.' + metaTypeName);
++ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName, metaTypeName);
+ }
+ Datatype inlineItemType = isInlineItemType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName)
+ : null;
+ Datatype inlineMetaType = isInlineMetaType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName)
+ : null;
+
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
sourceLoc);
+
+ // drop inline item and meta types
+ if (isInlineItemType && inlineItemType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName);
+ }
+ if (isInlineMetaType && inlineMetaType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName);
+ }
+
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e) {
if (bActiveTxn.booleanValue()) {
diff --cc asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 88017db,9b54562..ca515e5
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@@ -496,9 -464,9 +496,10 @@@ NodegroupDecl NodegroupSpecification()
DatasetDecl DatasetSpecification() throws ParseException:
{
- Pair<Identifier,Identifier> nameComponents = null;
+ Pair<DataverseName,Identifier> nameComponents = null;
boolean ifNotExists = false;
- Pair<Identifier,Identifier> typeComponents = null;
+ Pair<DataverseName,Identifier> typeComponents = null;
++ Pair<DataverseName,Identifier> metaTypeComponents = null;
String adapterName = null;
Map<String,String> properties = null;
FunctionSignature appliedFunction = null;
@@@ -508,7 -476,7 +509,6 @@@
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
Pair<Integer, List<String>> filterField = null;
- Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null);
- Pair<Identifier,Identifier> metaTypeComponents = null;
RecordConstructor withRecord = null;
}
{
diff --cc asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index 811a494,22753d0..c0ad1b8
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@@ -36,11 -36,9 +37,9 @@@ import org.apache.asterix.runtime.compr
public class DatasetDecl extends AbstractStatement {
protected final Identifier name;
- protected final Identifier dataverse;
+ protected final DataverseName dataverse;
- protected final DataverseName itemTypeDataverse;
- protected final Identifier itemTypeName;
- protected final DataverseName metaItemTypeDataverse;
- protected final Identifier metaItemTypeName;
+ protected final TypeExpression itemType;
+ protected final TypeExpression metaItemType;
protected final Identifier nodegroupName;
protected final DatasetType datasetType;
protected final IDatasetDetailsDecl datasetDetailsDecl;
@@@ -48,8 -46,7 +47,7 @@@
private AdmObjectNode withObjectNode;
protected final boolean ifNotExists;
- public DatasetDecl(DataverseName dataverse, Identifier name, DataverseName itemTypeDataverse,
- Identifier itemTypeName, DataverseName metaItemTypeDataverse, Identifier metaItemTypeName,
- public DatasetDecl(Identifier dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType,
++ public DatasetDecl(DataverseName dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType,
Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd,
RecordConstructor withRecord, boolean ifNotExists) throws CompilationException {
this.dataverse = dataverse;
@@@ -78,24 -73,12 +74,16 @@@
return name;
}
+ public DataverseName getDataverse() {
+ return dataverse;
+ }
+
- public Identifier getItemTypeName() {
- return itemTypeName;
+ public TypeExpression getItemType() {
+ return itemType;
}
- public DataverseName getItemTypeDataverse() {
- return itemTypeDataverse;
- }
-
- public Identifier getMetaItemTypeName() {
- return metaItemTypeName;
- }
-
- public DataverseName getMetaItemTypeDataverse() {
- return metaItemTypeDataverse;
+ public TypeExpression getMetaItemType() {
+ return metaItemType;
}
public Identifier getNodegroupName() {
diff --cc asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 53faff9,5aa45bc..17c94bd
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@@ -478,18 -428,11 +478,26 @@@ class SQLPPParser extends ScopeChecker
}
private SqlppParseException createUnexpectedTokenError() {
- return createUnexpectedTokenError(null);
- return createUnexpectedTokenError(token);
++ return createUnexpectedTokenError(token, null);
+ }
+
+ private SqlppParseException createUnexpectedTokenError(Token t) {
- return new SqlppParseException(getSourceLocation(t), "Unexpected token: " + LogRedactionUtil.userData(t.image));
++ return createUnexpectedTokenError(t, null);
+ }
+
+ private SqlppParseException createUnexpectedTokenError(String expected) {
- String message = "Unexpected token: " + LogRedactionUtil.userData(token.image) +
++ return createUnexpectedTokenError(token, expected);
++ }
++
++ private SqlppParseException createUnexpectedTokenError(Token t, String expected) {
++ String message = "Unexpected token: " + LogRedactionUtil.userData(t.image) +
+ (expected == null ? "" : ". Expected: " + LogRedactionUtil.userData(expected));
- return new SqlppParseException(getSourceLocation(token), message);
++ return new SqlppParseException(getSourceLocation(t), message);
+ }
+
+ private boolean laToken(int idx, int kind) {
+ Token t = getToken(idx);
+ return t.kind == kind;
}
private boolean laToken(int idx, int kind, String image) {
@@@ -749,12 -648,13 +757,10 @@@ DatasetDecl CreateDatasetStatement(Toke
DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
{
- Pair<Identifier,Identifier> nameComponents = null;
+ Pair<DataverseName,Identifier> nameComponents = null;
boolean ifNotExists = false;
- Pair<DataverseName,Identifier> typeComponents = null;
- String adapterName = null;
- Map<String,String> properties = null;
- FunctionSignature appliedFunction = null;
+ TypeExpression typeExpr = null;
+ TypeExpression metaTypeExpr = null;
- String adapterName = null;
- Map<String,String> properties = null;
- FunctionSignature appliedFunction = null;
Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
@@@ -764,84 -664,156 +770,150 @@@
RecordConstructor withRecord = null;
}
{
+ nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
++ typeExpr = DatasetTypeSpecification()
(
- <EXTERNAL> Dataset() nameComponents = QualifiedName()
- typeExpr = DatasetTypeSpecification()
- ifNotExists = IfNotExists()
- <USING> adapterName = AdapterName() properties = Configuration()
- ( <ON> nodeGroupName = Identifier() )?
- ( <HINTS> hints = Properties() )?
- ( <WITH> withRecord = RecordConstructor() )?
- {
- ExternalDetailsDecl edd = new ExternalDetailsDecl();
- edd.setAdapter(adapterName);
- edd.setProperties(properties);
- try{
- stmt = new DatasetDecl(nameComponents.first,
- nameComponents.second,
- typeExpr,
- null,
- nodeGroupName != null? new Identifier(nodeGroupName): null,
- hints,
- DatasetType.EXTERNAL,
- edd,
- withRecord,
- ifNotExists);
- } catch (CompilationException e){
- throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
+ { String name; }
+ <WITH>
+ name = Identifier()
+ {
+ if (!name.equalsIgnoreCase("meta")){
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "We can only support one additional associated field called \"meta\".");
}
- }
+ }
- <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN>
++ metaTypeExpr = DatasetTypeSpecification()
+ )?
+ ifNotExists = IfNotExists()
+ primaryKeyFields = PrimaryKey()
+ (<AUTOGENERATED> { autogenerated = true; } )?
+ (<ON> nodeGroupName = Identifier() )?
+ ( <HINTS> hints = Properties() )?
+ ( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )?
+ ( <WITH> withRecord = RecordConstructor() )?
+ {
+ if(filterField!=null && filterField.first!=0){
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "A filter field can only be a field in the main record of the dataset.");
+ }
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
+ filterField == null? null : filterField.second);
+ try {
- stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second,
- metaTypeComponents.first, metaTypeComponents.second,
++ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr,
+ nodeGroupName != null ? new Identifier(nodeGroupName) : null, hints, DatasetType.INTERNAL, idd, withRecord,
+ ifNotExists);
+ return addSourceLocation(stmt, startStmtToken);
+ } catch (CompilationException e) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
+ }
+ }
+}
- | ( <INTERNAL> )?
- Dataset() nameComponents = QualifiedName()
- typeExpr = DatasetTypeSpecification()
- (
- { String name; }
- <WITH>
- name = Identifier()
- {
- if (!name.equalsIgnoreCase("meta")){
- throw new SqlppParseException(getSourceLocation(startStmtToken),
- "We can only support one additional associated field called \"meta\".");
- }
- }
- metaTypeExpr = DatasetTypeSpecification()
- )?
- ifNotExists = IfNotExists()
- primaryKeyFields = PrimaryKey()
- (<AUTOGENERATED> { autogenerated = true; } )?
- (<ON> nodeGroupName = Identifier() )?
- ( <HINTS> hints = Properties() )?
- ( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )?
- ( <WITH> withRecord = RecordConstructor() )?
- {
- if(filterField!=null && filterField.first!=0){
- throw new SqlppParseException(getSourceLocation(startStmtToken),
- "A filter field can only be a field in the main record of the dataset.");
- }
- InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
- primaryKeyFields.first,
- autogenerated,
- filterField == null? null : filterField.second);
- try{
- stmt = new DatasetDecl(nameComponents.first,
- nameComponents.second,
- typeExpr,
- metaTypeExpr,
- nodeGroupName != null ? new Identifier(nodeGroupName) : null,
- hints,
- DatasetType.INTERNAL,
- idd,
- withRecord,
- ifNotExists);
- } catch (CompilationException e){
- throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
- }
- }
- )
- {
+DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseException:
+{
+ Pair<DataverseName,Identifier> nameComponents = null;
++ TypeExpression typeExpr = null;
+ boolean ifNotExists = false;
- Pair<DataverseName,Identifier> typeComponents = null;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ String nodeGroupName = null;
+ Map<String,String> hints = new HashMap<String,String>();
+ DatasetDecl stmt = null;
- Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null);
+ RecordConstructor withRecord = null;
+}
+{
+ nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
++ typeExpr = DatasetTypeSpecification()
+ ifNotExists = IfNotExists()
+ <USING> adapterName = AdapterName() properties = Configuration()
+ ( <ON> nodeGroupName = Identifier() )?
+ ( <HINTS> hints = Properties() )?
+ ( <WITH> withRecord = RecordConstructor() )?
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(properties);
+ try {
- stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second,
- metaTypeComponents.first, metaTypeComponents.second,
++ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, null,
+ nodeGroupName != null? new Identifier(nodeGroupName): null, hints, DatasetType.EXTERNAL, edd, withRecord,
+ ifNotExists);
return addSourceLocation(stmt, startStmtToken);
+ } catch (CompilationException e) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
}
+ }
}
+ TypeExpression DatasetTypeSpecification() throws ParseException:
+ {
+ TypeExpression typeExpr = null;
+ }
+ {
+ (
+ LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true)
+ | typeExpr = DatasetReferenceTypeSpecification()
+ )
+ {
+ return typeExpr;
+ }
+ }
+
+ TypeExpression DatasetReferenceTypeSpecification() throws ParseException:
+ {
+ TypeExpression typeExpr = null;
+ }
+ {
+ <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN>
+ {
+ return typeExpr;
+ }
+ }
+
+ TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException:
+ {
+ RecordTypeDefinition recordTypeDef = null;
+ RecordTypeDefinition.RecordKind recordKind = null;
+ Token recordKindToken = null;
+ }
+ {
+ <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
+ ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
+ {
+ if (recordKind == null) {
+ recordKind = RecordTypeDefinition.RecordKind.CLOSED;
+ } else if (!allowRecordKindModifier) {
+ throw createUnexpectedTokenError(recordKindToken);
+ }
+ recordTypeDef.setRecordKind(recordKind);
+ return recordTypeDef;
+ }
+ }
+
+ RecordTypeDefinition DatasetRecordTypeDef() throws ParseException:
+ {
+ RecordTypeDefinition recType = new RecordTypeDefinition();
+ }
+ {
+ DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )*
+ {
+ return recType;
+ }
+ }
+
+ void DatasetRecordField(RecordTypeDefinition recType) throws ParseException:
+ {
+ String fieldName;
+ TypeExpression type = null;
+ boolean isUnknownable = true;
+ }
+ {
+ fieldName = Identifier()
+ type = TypeReference() ( <NOT> <NULL> { isUnknownable = false; } )?
+ {
+ recType.addField(fieldName, type, isUnknownable);
+ }
+ }
+
RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
{
Token startToken = null;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 84166ce,a5084e0..78c026d
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@@ -590,4 -595,27 +596,12 @@@ public class DatasetUtil
return nodeGroup;
}
- // This doesn't work if the dataset or the dataverse name contains a '.'
- public static Pair<String, String> getDatasetInfo(MetadataProvider metadata, String datasetArg) {
- String first;
- String second;
- int i = datasetArg.indexOf('.');
- if (i > 0 && i < datasetArg.length() - 1) {
- first = datasetArg.substring(0, i);
- second = datasetArg.substring(i + 1);
- } else {
- first = metadata.getDefaultDataverse() == null ? null : metadata.getDefaultDataverse().getDataverseName();
- second = datasetArg;
- }
- return new Pair<>(first, second);
- }
-
+ public static String createInlineTypeName(String datasetName, boolean forMetaItemType) {
+ char typeChar = forMetaItemType ? 'm' : 'i';
+ return DATASET_INLINE_TYPE_PREFIX + typeChar + '$' + datasetName;
+ }
+
- public static boolean isInlineTypeName(Dataset dataset, String typeDataverseName, String typeName) {
++ public static boolean isInlineTypeName(Dataset dataset, DataverseName typeDataverseName, String typeName) {
+ return dataset.getDataverseName().equals(typeDataverseName) && typeName.startsWith(DATASET_INLINE_TYPE_PREFIX);
+ }
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
index 88d292b,baac712..1b88a06
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
@@@ -24,48 -22,27 +24,50 @@@ import org.apache.asterix.common.metada
import org.apache.asterix.common.metadata.LockList;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-public class MetadataLockUtil {
+public class MetadataLockUtil implements IMetadataLockUtil {
- private MetadataLockUtil() {
+ @Override
+ public void createDataverseBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName)
+ throws AlgebricksException {
+ lockMgr.acquireDataverseReadLock(locks, dataverseName);
+ }
+
+ @Override
+ public void dropDataverseBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName)
+ throws AlgebricksException {
+ lockMgr.acquireDataverseWriteLock(locks, dataverseName);
+ }
+
+ @Override
+ public void createDatasetBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName,
+ String datasetName, DataverseName itemTypeDataverseName, String itemTypeName,
+ DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName,
+ String compactionPolicyName, boolean isDefaultCompactionPolicy, Object datasetDetails)
+ throws AlgebricksException {
+ createDatasetBeginPre(lockMgr, locks, dataverseName, itemTypeDataverseName, itemTypeName,
+ metaItemTypeDataverseName, metaItemTypeName, nodeGroupName, compactionPolicyName,
+ isDefaultCompactionPolicy);
+ lockMgr.acquireDatasetWriteLock(locks, dataverseName, datasetName);
}
- public static void createDatasetBegin(IMetadataLockManager lockMgr, LockList locks, String dataverseName,
- String itemTypeDataverseName, String itemTypeFullyQualifiedName, String metaItemTypeDataverseName,
- String metaItemTypeFullyQualifiedName, String nodeGroupName, String compactionPolicyName,
- String datasetFullyQualifiedName, boolean isDefaultCompactionPolicy) throws AlgebricksException {
+ protected final void createDatasetBeginPre(IMetadataLockManager lockMgr, LockList locks,
+ DataverseName dataverseName, DataverseName itemTypeDataverseName, String itemTypeName,
+ DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName,
+ String compactionPolicyName, boolean isDefaultCompactionPolicy) throws AlgebricksException {
lockMgr.acquireDataverseReadLock(locks, dataverseName);
-- if (!dataverseName.equals(itemTypeDataverseName)) {
++ if (itemTypeDataverseName != null && !dataverseName.equals(itemTypeDataverseName)) {
lockMgr.acquireDataverseReadLock(locks, itemTypeDataverseName);
}
if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(dataverseName)
&& !metaItemTypeDataverseName.equals(itemTypeDataverseName)) {
lockMgr.acquireDataverseReadLock(locks, metaItemTypeDataverseName);
}
- lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName);
- lockMgr.acquireDataTypeReadLock(locks, itemTypeFullyQualifiedName);
- if (metaItemTypeFullyQualifiedName != null
- && !metaItemTypeFullyQualifiedName.equals(itemTypeFullyQualifiedName)) {
- lockMgr.acquireDataTypeReadLock(locks, metaItemTypeFullyQualifiedName);
++ if (itemTypeName != null) {
++ lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName);
++ }
+ if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(itemTypeDataverseName)
+ && !metaItemTypeName.equals(itemTypeName)) {
+ lockMgr.acquireDataTypeReadLock(locks, metaItemTypeDataverseName, metaItemTypeName);
}
if (nodeGroupName != null) {
lockMgr.acquireNodeGroupReadLock(locks, nodeGroupName);
[asterixdb] 01/03: [NO ISSUE] Minor cleanup in external datasets
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 74cecd932af78831c7fe74a52658ae6844606abe
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Mar 31 10:35:40 2020 -0700
[NO ISSUE] Minor cleanup in external datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Introduce createExternalDataSourceFactory() and
createDataParserFactory() in GenericAdapterFactory
for better extensibility by products
- Cleanup ExternalDataConstants
Change-Id: Ifb56ffce34a0b62ece3340582331bb78f2f5c5ca
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5525
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../runtimets/results/feeds/feeds_01/feeds_01.1.adm | 2 +-
.../adapter/factory/GenericAdapterFactory.java | 20 +++++++++++++++-----
.../util/ExternalDataCompatibilityUtils.java | 7 ++-----
.../asterix/external/util/ExternalDataConstants.java | 10 ----------
4 files changed, 18 insertions(+), 21 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
index 1dc31dc..3ab469e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
@@ -1 +1 @@
-{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "linkName", "Value": "localfs" }, [...]
\ No newline at end of file
+{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "type-name", "Value": "TweetType" [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index d081e56..078316c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataFlowController;
@@ -122,7 +123,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager)
throws HyracksDataException, AlgebricksException {
if (dataSourceFactory == null) {
- dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
+ dataSourceFactory = createExternalDataSourceFactory(configuration, libraryManager);
// create and configure parser factory
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
@@ -131,7 +132,7 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
}
if (dataParserFactory == null) {
// create and configure parser factory
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
+ dataParserFactory = createDataParserFactory(configuration, libraryManager);
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
@@ -144,14 +145,13 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
this.configuration = configuration;
IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext();
ExternalDataUtils.validateDataSourceParameters(configuration);
- dataSourceFactory =
- DatasourceFactoryProvider.getExternalDataSourceFactory(appCtx.getLibraryManager(), configuration);
+ dataSourceFactory = createExternalDataSourceFactory(configuration, appCtx.getLibraryManager());
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
}
dataSourceFactory.configure(serviceContext, configuration);
ExternalDataUtils.validateDataParserParameters(configuration);
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(appCtx.getLibraryManager(), configuration);
+ dataParserFactory = createDataParserFactory(configuration, appCtx.getLibraryManager());
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
@@ -222,4 +222,14 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda
dataParserFactory.configure(Collections.emptyMap());
configuration = Collections.emptyMap();
}
+
+ protected IExternalDataSourceFactory createExternalDataSourceFactory(Map<String, String> configuration,
+ ILibraryManager libraryManager) throws HyracksDataException, AsterixException {
+ return DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
+ }
+
+ protected IDataParserFactory createDataParserFactory(Map<String, String> configuration,
+ ILibraryManager libraryManager) throws AsterixException {
+ return ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
index 77cbb96..e222e99 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
@@ -60,11 +60,8 @@ public class ExternalDataCompatibilityUtils {
}
public static void prepare(String adapterName, Map<String, String> configuration) {
- // Adapter name in some cases can carry the link name for external datasets, always add it to configuration
- configuration.put(ExternalDataConstants.KEY_LINK_NAME, adapterName);
-
- if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) { // SThree
- configuration.put(ExternalDataConstants.KEY_READER, adapterName); // myAwsLink
+ if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) {
+ configuration.put(ExternalDataConstants.KEY_READER, adapterName);
}
if (!configuration.containsKey(ExternalDataConstants.KEY_PARSER)) {
if (configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index e44144a..26f5402 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -231,16 +231,6 @@ public class ExternalDataConstants {
public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
- // TODO(Hussain): Move link related items to a different place
- /**
- * Common external link fields
- */
- public static final String KEY_DATAVERSE_NAME = "dataverseName";
- public static final String KEY_LINK_NAME = "linkName";
- public static final String KEY_LINK_TYPE = "linkType";
- public static final String[] KEY_EXTERNAL_DATASET_REQUIRED_CONNECTION_PARAMETERS =
- new String[] { KEY_DATAVERSE_NAME, KEY_LINK_NAME, KEY_LINK_TYPE };
-
public static class AwsS3Constants {
public static final String REGION_FIELD_NAME = "region";
public static final String ACCESS_KEY_FIELD_NAME = "accessKey";
[asterixdb] 02/03: [NO ISSUE][COMP] CREATE DATASET with inline type
definition
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3e2623cf64827e891ecd8c6dea13d03f5599c6a5
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Wed Apr 1 11:20:04 2020 -0700
[NO ISSUE][COMP] CREATE DATASET with inline type definition
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Support inline type specification in CREATE DATASET statement:
CREATE DATASET customer(cid INTEGER NOT NULL, first_name STRING) ...
- Add testcases
Change-Id: Ic88ccacd016a6144f1b05a2f79a07b2a980d9c9b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5524
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../asterix/app/translator/QueryTranslator.java | 147 ++++++++++++++++++---
.../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++++++++++
.../create-dataset-inline-type-1.2.query.sqlpp | 22 +++
.../create-dataset-inline-type-1.3.ddl.sqlpp | 30 +++++
.../create-dataset-inline-type-1.4.query.sqlpp | 26 ++++
.../create-dataset-inline-type-1.2.adm | 21 +++
.../create-dataset-inline-type-1.4.adm | 1 +
.../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +-
.../asterix/lang/common/statement/DatasetDecl.java | 64 ++-------
.../lang/common/visitor/FormatPrintVisitor.java | 12 +-
.../lang/common/visitor/QueryPrintVisitor.java | 15 ++-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 128 ++++++++++++++----
.../metadata/bootstrap/MetadataRecordTypes.java | 2 +-
.../DatasetTupleTranslator.java | 4 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 21 ++-
.../asterix/metadata/utils/MetadataUtil.java | 1 +
18 files changed, 533 insertions(+), 127 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index b8a048d..a440ae4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -91,6 +91,8 @@ import org.apache.asterix.lang.common.base.IRewriterFactory;
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -560,10 +562,45 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
String dataverseName = getActiveDataverse(dd.getDataverse());
String datasetName = dd.getName().getValue();
DatasetType dsType = dd.getDatasetType();
- String itemTypeDataverseName = getActiveDataverse(dd.getItemTypeDataverse());
- String itemTypeName = dd.getItemTypeName().getValue();
- String metaItemTypeDataverseName = getActiveDataverse(dd.getMetaItemTypeDataverse());
- String metaItemTypeName = dd.getMetaItemTypeName().getValue();
+ TypeExpression itemTypeExpr = dd.getItemType();
+ String itemTypeDataverseName = null, itemTypeName = null, itemTypeFullyQualifiedName = null;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression itemTypeRefExpr = (TypeReferenceExpression) itemTypeExpr;
+ Identifier itemTypeDataverseIdent = itemTypeRefExpr.getIdent().first;
+ itemTypeDataverseName = itemTypeDataverseIdent != null && itemTypeDataverseIdent.getValue() != null
+ ? itemTypeDataverseIdent.getValue() : dataverseName;
+ itemTypeName = itemTypeRefExpr.getIdent().second.getValue();
+ itemTypeFullyQualifiedName = itemTypeDataverseName + '.' + itemTypeName;
+ break;
+ case RECORD:
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(itemTypeExpr.getTypeKind()));
+ }
+
+ TypeExpression metaItemTypeExpr = dd.getMetaItemType();
+ String metaItemTypeDataverseName = null, metaItemTypeName = null, metaItemTypeFullyQualifiedName = null;
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression metaItemTypeRefExpr = (TypeReferenceExpression) metaItemTypeExpr;
+ Identifier metaItemTypeDataverseIdent = metaItemTypeRefExpr.getIdent().first;
+ metaItemTypeDataverseName =
+ metaItemTypeDataverseIdent != null && metaItemTypeDataverseIdent.getValue() != null
+ ? metaItemTypeDataverseIdent.getValue() : dataverseName;
+ metaItemTypeName = metaItemTypeRefExpr.getIdent().second.getValue();
+ metaItemTypeFullyQualifiedName = metaItemTypeDataverseName + '.' + metaItemTypeName;
+ break;
+ case RECORD:
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
+ }
+
Identifier ngNameId = dd.getNodegroupName();
String nodegroupName = ngNameId == null ? null : ngNameId.getValue();
String compactionPolicy = dd.getCompactionPolicy();
@@ -575,9 +612,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
MetadataLockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName,
- itemTypeDataverseName, itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName,
- metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy,
- dataverseName + "." + datasetName, defaultCompactionPolicy);
+ itemTypeDataverseName, itemTypeFullyQualifiedName, metaItemTypeDataverseName,
+ metaItemTypeFullyQualifiedName, nodegroupName, compactionPolicy, dataverseName + "." + datasetName,
+ defaultCompactionPolicy);
Dataset dataset = null;
try {
IDatasetDetails datasetDetails = null;
@@ -590,10 +627,23 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName);
}
}
- Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- itemTypeDataverseName, itemTypeName);
- if (dt == null) {
- throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeName);
+ IAType itemType;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ itemType = metadataProvider.findType(itemTypeDataverseName, itemTypeName);
+ break;
+ case RECORD:
+ itemTypeDataverseName = dataverseName;
+ itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false);
+ MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
+ itemTypeDataverseName + "." + itemTypeName);
+ itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(itemTypeDataverseName, itemTypeName, itemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(itemTypeExpr.getTypeKind()));
}
String ngName = ngNameId != null ? ngNameId.getValue()
: configureNodegroupForDataset(appCtx, dd.getHints(), dataverseName, datasetName, metadataProvider,
@@ -607,19 +657,35 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
switch (dd.getDatasetType()) {
case INTERNAL:
- IAType itemType = dt.getDatatype();
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Dataset type has to be a record type.");
}
IAType metaItemType = null;
- if (metaItemTypeDataverseName != null && metaItemTypeName != null) {
- metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
- }
- if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.OBJECT) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Dataset meta type has to be a record type.");
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
+ if (metaItemType.getTypeTag() != ATypeTag.OBJECT) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Dataset meta type has to be a record type.");
+ }
+ break;
+ case RECORD:
+ metaItemTypeDataverseName = dataverseName;
+ metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true);
+ MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
+ metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName);
+ metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName,
+ metaItemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
}
ARecordType metaRecType = (ARecordType) metaItemType;
@@ -1234,10 +1300,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Cannot redefine builtin type " + typeName + ".");
} else {
- Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx,
- stmtCreateType.getTypeDef(), stmtCreateType.getIdent().getValue(), dataverseName);
- TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
- IAType type = typeMap.get(typeSignature);
+ IAType type = translateType(dataverseName, typeName, stmtCreateType.getTypeDef(), mdTxnCtx);
MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false));
}
}
@@ -1250,6 +1313,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
}
+ private IAType translateType(String dataverseName, String typeName, TypeExpression typeDef,
+ MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
+ Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, typeDef, typeName, dataverseName);
+ TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
+ return typeMap.get(typeSignature);
+ }
+
protected void handleDataverseDropStatement(MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception {
DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt;
@@ -1459,8 +1529,41 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
}
validateDatasetState(metadataProvider, ds, sourceLoc);
+
+ // prepare to drop item and meta types if they were created as inline types
+ String itemTypeDataverseName = ds.getItemTypeDataverseName();
+ String itemTypeName = ds.getItemTypeName();
+ boolean isInlineItemType = DatasetUtil.isInlineTypeName(ds, itemTypeDataverseName, itemTypeName);
+ if (isInlineItemType) {
+ MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
+ itemTypeDataverseName + '.' + itemTypeName);
+ }
+ String metaTypeDataverseName = ds.getMetaItemTypeDataverseName();
+ String metaTypeName = ds.getMetaItemTypeName();
+ boolean isInlineMetaType =
+ metaTypeName != null && DatasetUtil.isInlineTypeName(ds, metaTypeDataverseName, metaTypeName);
+ if (isInlineMetaType) {
+ MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName,
+ metaTypeDataverseName + '.' + metaTypeName);
+ }
+ Datatype inlineItemType = isInlineItemType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName)
+ : null;
+ Datatype inlineMetaType = isInlineMetaType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName)
+ : null;
+
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
sourceLoc);
+
+ // drop inline item and meta types
+ if (isInlineItemType && inlineItemType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName);
+ }
+ if (isInlineMetaType && inlineMetaType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName);
+ }
+
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e) {
if (bActiveTxn.booleanValue()) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp
new file mode 100644
index 0000000..6230d4b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+/* Metadata test function */
+
+CREATE FUNCTION listMetadata() {
+ SELECT "Dataset" AS en, d.DatasetName, d.DatatypeDataverseName, d.DatatypeName,
+ d.MetatypeDataverseName, d.MetatypeName
+ FROM Metadata.`Dataset` d
+ WHERE d.DataverseName = "test"
+ UNION ALL
+ SELECT "Datatype" AS en, dt.DatatypeName, dt.Derived
+ FROM Metadata.`Datatype` dt
+ WHERE dt.DataverseName = "test"
+ ORDER BY en, DatasetName, DatatypeName
+};
+
+/* Internal datasets */
+
+CREATE DATASET A_Customers_Default_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) PRIMARY KEY c_custkey ;
+
+CREATE DATASET A_Customers_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) CLOSED TYPE PRIMARY KEY c_custkey;
+
+CREATE DATASET A_Customers_Open(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) OPEN TYPE PRIMARY KEY c_custkey;
+
+/* External datasets */
+
+CREATE EXTERNAL DATASET B_Orders_Default_Closed(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+)
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+CREATE EXTERNAL DATASET B_Orders_Closed(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+) CLOSED TYPE
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+CREATE EXTERNAL DATASET B_Orders_Open(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+) OPEN TYPE
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+/* Internal datasets with inline META type */
+
+CREATE DATASET C_Customers_Meta_Default_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+)
+WITH META(c_x integer not null, c_y integer)
+PRIMARY KEY c_custkey ;
+
+CREATE DATASET C_Customers_Meta_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) CLOSED TYPE
+WITH META(
+ c_x integer not null,
+ c_y integer
+) CLOSED TYPE
+PRIMARY KEY c_custkey;
+
+CREATE DATASET C_Customers_Meta_Open(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) OPEN TYPE
+WITH META(
+ c_x integer not null,
+ c_y integer
+) OPEN TYPE
+PRIMARY KEY c_custkey;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp
new file mode 100644
index 0000000..52a6324
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+listMetadata();
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp
new file mode 100644
index 0000000..8a08888
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+DROP DATASET A_Customers_Default_Closed;
+DROP DATASET A_Customers_Closed;
+DROP DATASET A_Customers_Open;
+DROP DATASET B_Orders_Default_Closed;
+DROP DATASET B_Orders_Closed;
+DROP DATASET B_Orders_Open;
+DROP DATASET C_Customers_Meta_Default_Closed;
+DROP DATASET C_Customers_Meta_Closed;
+DROP DATASET C_Customers_Meta_Open;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp
new file mode 100644
index 0000000..ad16400
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that inline types are deleted when dataset is dropped
+ */
+
+USE test;
+
+array_count(listMetadata());
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm
new file mode 100644
index 0000000..8a539c8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm
@@ -0,0 +1,21 @@
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Closed", "DatasetName": "A_Customers_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "DatasetName": "A_Customers_Default_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Open", "DatasetName": "A_Customers_Open", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Closed", "DatasetName": "B_Orders_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "DatasetName": "B_Orders_Default_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Open", "DatasetName": "B_Orders_Open", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "DatasetName": "C_Customers_Meta_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Closed" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "DatasetName": "C_Customers_Meta_Default_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Default_Closed" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "DatasetName": "C_Customers_Meta_Open", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Open" }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "Field [...]
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate" [...]
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "FieldTyp [...]
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
index c3a4877..ffe5bc3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
@@ -17,7 +17,7 @@ TypeDecl tchrType [
dept : string
}
]
-DatasetDecl ugdstd(stdType) partitioned by [[id]]
-DatasetDecl gdstd(stdType) partitioned by [[id]]
+DatasetDecl ugdstd(student.stdType) partitioned by [[id]]
+DatasetDecl gdstd(student.stdType) partitioned by [[id]]
DatasetDecl prof(tchrType) partitioned by [[id]]
-DatasetDecl pstdoc(tchrType) partitioned by [[id]]
+DatasetDecl pstdoc(tchrType) partitioned by [[id]]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 5ef5808..0d17fb1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3860,6 +3860,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-inline-type-1">
+ <output-dir compare="Text">create-dataset-inline-type-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
<compilation-unit name="drop-primary-index">
<output-dir compare="Text">drop-primary-index</output-dir>
<expected-error>Cannot drop index "ds". Drop dataset "ds" to remove this index</expected-error>
diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 4c8820f..9b54562 100644
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -476,7 +476,7 @@ DatasetDecl DatasetSpecification() throws ParseException:
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
Pair<Integer, List<String>> filterField = null;
- Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
+ Pair<Identifier,Identifier> metaTypeComponents = null;
RecordConstructor withRecord = null;
}
{
@@ -495,10 +495,8 @@ DatasetDecl DatasetSpecification() throws ParseException:
try{
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ new TypeReferenceExpression(typeComponents),
+ null,
nodeGroupName != null? new Identifier(nodeGroupName): null,
hints,
DatasetType.EXTERNAL,
@@ -542,10 +540,8 @@ DatasetDecl DatasetSpecification() throws ParseException:
try{
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ new TypeReferenceExpression(typeComponents),
+ metaTypeComponents != null ? new TypeReferenceExpression(metaTypeComponents) : null,
nodeGroupName != null ? new Identifier(nodeGroupName) : null,
hints,
DatasetType.INTERNAL,
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index 45fc33a..22753d0 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@ -25,6 +25,7 @@ import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.util.ConfigurationUtil;
import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
@@ -36,10 +37,8 @@ import org.apache.asterix.runtime.compression.CompressionManager;
public class DatasetDecl extends AbstractStatement {
protected final Identifier name;
protected final Identifier dataverse;
- protected final Identifier itemTypeDataverse;
- protected final Identifier itemTypeName;
- protected final Identifier metaItemTypeDataverse;
- protected final Identifier metaItemTypeName;
+ protected final TypeExpression itemType;
+ protected final TypeExpression metaItemType;
protected final Identifier nodegroupName;
protected final DatasetType datasetType;
protected final IDatasetDetailsDecl datasetDetailsDecl;
@@ -47,24 +46,13 @@ public class DatasetDecl extends AbstractStatement {
private AdmObjectNode withObjectNode;
protected final boolean ifNotExists;
- public DatasetDecl(Identifier dataverse, Identifier name, Identifier itemTypeDataverse, Identifier itemTypeName,
- Identifier metaItemTypeDataverse, Identifier metaItemTypeName, Identifier nodeGroupName,
- Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, RecordConstructor withRecord,
- boolean ifNotExists) throws CompilationException {
+ public DatasetDecl(Identifier dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType,
+ Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd,
+ RecordConstructor withRecord, boolean ifNotExists) throws CompilationException {
this.dataverse = dataverse;
this.name = name;
- this.itemTypeName = itemTypeName;
- if (itemTypeDataverse.getValue() == null) {
- this.itemTypeDataverse = dataverse;
- } else {
- this.itemTypeDataverse = itemTypeDataverse;
- }
- this.metaItemTypeName = metaItemTypeName;
- if (metaItemTypeDataverse == null || metaItemTypeDataverse.getValue() == null) {
- this.metaItemTypeDataverse = dataverse;
- } else {
- this.metaItemTypeDataverse = metaItemTypeDataverse;
- }
+ this.itemType = itemType;
+ this.metaItemType = metaItemType;
this.nodegroupName = nodeGroupName;
this.hints = hints;
this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord, datasetType);
@@ -85,40 +73,12 @@ public class DatasetDecl extends AbstractStatement {
return name;
}
- public Identifier getItemTypeName() {
- return itemTypeName;
- }
-
- public Identifier getItemTypeDataverse() {
- return itemTypeDataverse;
+ public TypeExpression getItemType() {
+ return itemType;
}
- public String getQualifiedTypeName() {
- if (itemTypeDataverse == dataverse) {
- return itemTypeName.getValue();
- } else {
- return itemTypeDataverse.getValue() + "." + itemTypeName.getValue();
- }
- }
-
- public Identifier getMetaName() {
- return name;
- }
-
- public Identifier getMetaItemTypeName() {
- return metaItemTypeName == null ? new Identifier() : metaItemTypeName;
- }
-
- public Identifier getMetaItemTypeDataverse() {
- return metaItemTypeDataverse == null ? new Identifier() : metaItemTypeDataverse;
- }
-
- public String getQualifiedMetaTypeName() {
- if (metaItemTypeDataverse == dataverse) {
- return metaItemTypeName.getValue();
- } else {
- return metaItemTypeDataverse.getValue() + "." + metaItemTypeName.getValue();
- }
+ public TypeExpression getMetaItemType() {
+ return metaItemType;
}
public Identifier getNodegroupName() {
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 6b734dd..c123348 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -457,16 +457,18 @@ public class FormatPrintVisitor implements ILangVisitor<Void, Integer> {
public Void visit(DatasetDecl dd, Integer step) throws CompilationException {
if (dd.getDatasetType() == DatasetType.INTERNAL) {
out.print(skip(step) + "create " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName())
- + generateIfNotExists(dd.getIfNotExists()) + "(" + dd.getQualifiedTypeName() + ")"
- + " primary key ");
+ + generateIfNotExists(dd.getIfNotExists()) + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ") primary key ");
printDelimitedKeys(((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(), ",");
if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) {
out.print(" autogenerated ");
}
} else if (dd.getDatasetType() == DatasetType.EXTERNAL) {
- out.print(
- skip(step) + "create external " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName())
- + "(" + dd.getQualifiedTypeName() + ")" + generateIfNotExists(dd.getIfNotExists()));
+ out.print(skip(step) + "create external " + datasetSymbol
+ + generateFullName(dd.getDataverse(), dd.getName()) + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ")" + generateIfNotExists(dd.getIfNotExists()));
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
out.print(" using " + revertStringToQuoted(externalDetails.getAdapter()));
printConfiguration(externalDetails.getProperties());
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
index ff55880..5b8c9c0 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
@@ -388,15 +388,18 @@ public abstract class QueryPrintVisitor extends AbstractQueryExpressionVisitor<V
@Override
public Void visit(DatasetDecl dd, Integer step) throws CompilationException {
if (dd.getDatasetType() == DatasetType.INTERNAL) {
- String line = skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")"
- + " partitioned by " + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs();
+ out.print(skip(step) + "DatasetDecl " + dd.getName() + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ") partitioned by "
+ + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs());
if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) {
- line += " [autogenerated]";
+ out.print(" [autogenerated]");
}
- out.println(line);
+ out.println();
} else if (dd.getDatasetType() == DatasetType.EXTERNAL) {
- out.println(skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")"
- + "is an external dataset");
+ out.print(skip(step) + "DatasetDecl " + dd.getName() + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.println(skip(step) + ")is an external dataset");
}
return null;
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index e0a4341..5aa45bc 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -335,15 +335,15 @@ class SQLPPParser extends ScopeChecker implements IParser {
hintCollector.clear();
try {
return parseFunction.parse();
+ } catch (SqlppParseException e) {
+ throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e)));
+ } catch (ParseException e) {
+ throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
} catch (Error e) {
// this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled
// by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "");
throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(msg));
- } catch (SqlppParseException e) {
- throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e)));
- } catch (ParseException e) {
- throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
} finally {
reportUnclaimedHints();
}
@@ -428,7 +428,11 @@ class SQLPPParser extends ScopeChecker implements IParser {
}
private SqlppParseException createUnexpectedTokenError() {
- return new SqlppParseException(getSourceLocation(token), "Unexpected token: " + LogRedactionUtil.userData(token.image));
+ return createUnexpectedTokenError(token);
+ }
+
+ private SqlppParseException createUnexpectedTokenError(Token t) {
+ return new SqlppParseException(getSourceLocation(t), "Unexpected token: " + LogRedactionUtil.userData(t.image));
}
private boolean laToken(int idx, int kind, String image) {
@@ -646,7 +650,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
{
Pair<Identifier,Identifier> nameComponents = null;
boolean ifNotExists = false;
- Pair<Identifier,Identifier> typeComponents = null;
+ TypeExpression typeExpr = null;
+ TypeExpression metaTypeExpr = null;
String adapterName = null;
Map<String,String> properties = null;
FunctionSignature appliedFunction = null;
@@ -656,13 +661,12 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
DatasetDecl stmt = null;
boolean autogenerated = false;
Pair<Integer, List<String>> filterField = null;
- Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
RecordConstructor withRecord = null;
}
{
(
<EXTERNAL> Dataset() nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+ typeExpr = DatasetTypeSpecification()
ifNotExists = IfNotExists()
<USING> adapterName = AdapterName() properties = Configuration()
( <ON> nodeGroupName = Identifier() )?
@@ -675,10 +679,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
try{
stmt = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ typeExpr,
+ null,
nodeGroupName != null? new Identifier(nodeGroupName): null,
hints,
DatasetType.EXTERNAL,
@@ -692,7 +694,7 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
| ( <INTERNAL> )?
Dataset() nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+ typeExpr = DatasetTypeSpecification()
(
{ String name; }
<WITH>
@@ -703,7 +705,7 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
"We can only support one additional associated field called \"meta\".");
}
}
- <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN>
+ metaTypeExpr = DatasetTypeSpecification()
)?
ifNotExists = IfNotExists()
primaryKeyFields = PrimaryKey()
@@ -724,10 +726,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
try{
stmt = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ typeExpr,
+ metaTypeExpr,
nodeGroupName != null ? new Identifier(nodeGroupName) : null,
hints,
DatasetType.INTERNAL,
@@ -744,6 +744,76 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
}
}
+TypeExpression DatasetTypeSpecification() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true)
+ | typeExpr = DatasetReferenceTypeSpecification()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression DatasetReferenceTypeSpecification() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN>
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException:
+{
+ RecordTypeDefinition recordTypeDef = null;
+ RecordTypeDefinition.RecordKind recordKind = null;
+ Token recordKindToken = null;
+}
+{
+ <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
+ ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
+ {
+ if (recordKind == null) {
+ recordKind = RecordTypeDefinition.RecordKind.CLOSED;
+ } else if (!allowRecordKindModifier) {
+ throw createUnexpectedTokenError(recordKindToken);
+ }
+ recordTypeDef.setRecordKind(recordKind);
+ return recordTypeDef;
+ }
+}
+
+RecordTypeDefinition DatasetRecordTypeDef() throws ParseException:
+{
+ RecordTypeDefinition recType = new RecordTypeDefinition();
+}
+{
+ DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )*
+ {
+ return recType;
+ }
+}
+
+void DatasetRecordField(RecordTypeDefinition recType) throws ParseException:
+{
+ String fieldName;
+ TypeExpression type = null;
+ boolean isUnknownable = true;
+}
+{
+ fieldName = Identifier()
+ type = TypeReference() ( <NOT> <NULL> { isUnknownable = false; } )?
+ {
+ recType.addField(fieldName, type, isUnknownable);
+ }
+}
+
RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
{
Token startToken = null;
@@ -1575,15 +1645,28 @@ TypeExpression TypeExpr() throws ParseException:
}
}
+RecordTypeDefinition.RecordKind RecordTypeKind() throws ParseException:
+{
+ RecordTypeDefinition.RecordKind recordKind = null;
+}
+{
+ (
+ <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
+ | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; }
+ )
+ {
+ return recordKind;
+ }
+}
+
RecordTypeDefinition RecordTypeDef() throws ParseException:
{
Token startToken = null;
RecordTypeDefinition recType = new RecordTypeDefinition();
- RecordTypeDefinition.RecordKind recordKind = null;
+ RecordTypeDefinition.RecordKind recordKind = RecordTypeDefinition.RecordKind.OPEN;
}
{
- ( <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
- | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
+ ( recordKind = RecordTypeKind() )?
<LEFTBRACE>
{
startToken = token;
@@ -1610,9 +1693,6 @@ RecordTypeDefinition RecordTypeDef() throws ParseException:
)?
<RIGHTBRACE>
{
- if (recordKind == null) {
- recordKind = RecordTypeDefinition.RecordKind.OPEN;
- }
recType.setRecordKind(recordKind);
return addSourceLocation(recType, startToken);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
index d9309d9..7fdbfcf 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -74,7 +74,7 @@ public final class MetadataRecordTypes {
public static final String FIELD_NAME_KIND = "Kind";
public static final String FIELD_NAME_LANGUAGE = "Language";
public static final String FIELD_NAME_LAST_REFRESH_TIME = "LastRefreshTime";
- public static final String FIELD_NAME_METADATA_DATAVERSE = "MetatypeDataverseName";
+ public static final String FIELD_NAME_METATYPE_DATAVERSE = "MetatypeDataverseName";
public static final String FIELD_NAME_METATYPE_NAME = "MetatypeName";
public static final String FIELD_NAME_NAME = "Name";
public static final String FIELD_NAME_NODE_NAME = "NodeName";
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 7f8b9bf..dae6152 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -250,7 +250,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
String metaTypeDataverseName = null;
String metaTypeName = null;
int metaTypeDataverseNameIndex =
- datasetRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_METADATA_DATAVERSE);
+ datasetRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_METATYPE_DATAVERSE);
if (metaTypeDataverseNameIndex >= 0) {
metaTypeDataverseName =
((AString) datasetRecord.getValueByPos(metaTypeDataverseNameIndex)).getStringValue();
@@ -430,7 +430,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
if (dataset.hasMetaPart()) {
// write open field 1, the meta item type Dataverse name.
fieldName.reset();
- aString.setValue(MetadataRecordTypes.FIELD_NAME_METADATA_DATAVERSE);
+ aString.setValue(MetadataRecordTypes.FIELD_NAME_METATYPE_DATAVERSE);
stringSerde.serialize(aString, fieldName.getDataOutput());
fieldValue.reset();
aString.setValue(dataset.getMetaItemTypeDataverseName());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4b7d359..a5084e0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -99,6 +99,8 @@ public class DatasetUtil {
*/
public static final byte OP_UPSERT = 0x03;
+ private static final String DATASET_INLINE_TYPE_PREFIX = "$d$t$";
+
private DatasetUtil() {
}
@@ -207,9 +209,13 @@ public class DatasetUtil {
* field is actually a key by making sure the field is coming from the right record (data record or meta record),
* e.g. if the field name happens to be equal to the key name but the field is coming from the data record while
* the key is coming from the meta record.
- * @param keySourceIndicator indicates where the key is coming from, 1 from meta record, 0 from data record
- * @param keyIndex the key index we're checking the field against
- * @param fieldFromMeta whether the field is coming from the meta record or the data record
+ *
+ * @param keySourceIndicator
+ * indicates where the key is coming from, 1 from meta record, 0 from data record
+ * @param keyIndex
+ * the key index we're checking the field against
+ * @param fieldFromMeta
+ * whether the field is coming from the meta record or the data record
* @return true if the key source matches the field source. Otherwise, false.
*/
private static boolean keySourceMatches(List<Integer> keySourceIndicator, int keyIndex, boolean fieldFromMeta) {
@@ -603,4 +609,13 @@ public class DatasetUtil {
}
return new Pair<>(first, second);
}
+
+ public static String createInlineTypeName(String datasetName, boolean forMetaItemType) {
+ char typeChar = forMetaItemType ? 'm' : 'i';
+ return DATASET_INLINE_TYPE_PREFIX + typeChar + '$' + datasetName;
+ }
+
+ public static boolean isInlineTypeName(Dataset dataset, String typeDataverseName, String typeName) {
+ return dataset.getDataverseName().equals(typeDataverseName) && typeName.startsWith(DATASET_INLINE_TYPE_PREFIX);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
index e5d4721..3436b44 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
@@ -43,4 +43,5 @@ public class MetadataUtil {
int idx = datasetName.indexOf('.');
return datasetName.substring(0, idx);
}
+
}