You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/10/24 09:48:52 UTC
carbondata git commit: [CARBONDATA-1609] Thrift Changes to support
Pre-aggregate
Repository: carbondata
Updated Branches:
refs/heads/pre-aggregate 5a67c98f1 -> 334aa1ccd
[CARBONDATA-1609] Thrift Changes to support Pre-aggregate
This closes #1425
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/334aa1cc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/334aa1cc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/334aa1cc
Branch: refs/heads/pre-aggregate
Commit: 334aa1ccd696c05b7f84386186e3c111183da807
Parents: 5a67c98
Author: kumarvishal <ku...@gmail.com>
Authored: Sat Oct 21 16:31:15 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Oct 24 15:18:37 2017 +0530
----------------------------------------------------------------------
.../metadata/converter/SchemaConverter.java | 10 ++
.../ThriftWrapperSchemaConverterImpl.java | 147 +++++++++++++++++--
.../core/metadata/schema/table/CarbonTable.java | 24 +--
.../metadata/schema/table/DataMapSchema.java | 121 +++++++++++++++
.../schema/table/RelationIdentifier.java | 88 +++++++++++
.../core/metadata/schema/table/TableInfo.java | 79 +++++++++-
.../core/metadata/schema/table/TableSchema.java | 26 ++++
.../schema/table/column/ColumnSchema.java | 48 +++++-
.../util/AbstractDataFileFooterConverter.java | 28 ++++
format/src/main/thrift/schema.thrift | 31 ++++
10 files changed, 576 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java
index 4faa017..bfbb6f7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.metadata.converter;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -100,4 +101,13 @@ public interface SchemaConverter {
*/
TableInfo fromExternalToWrapperTableInfo(org.apache.carbondata.format.TableInfo externalTableInfo,
String dbName, String tableName, String storePath);
+
+ /**
+ * method to convert thrift datamap schema object to wrapper
+ * data map object
+ * @param thriftchildSchema
+ * @return DataMapSchema
+ */
+ DataMapSchema fromExternalToWrapperDataMapSchema(
+ org.apache.carbondata.format.DataMapSchema thriftchildSchema);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 7faa7e6..ed00b06 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -30,9 +30,12 @@ import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
/**
* Thrift schema to carbon schema converter and vice versa
@@ -40,11 +43,11 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
/* (non-Javadoc)
- * Converts from wrapper to thrift schema evolution entry
- */
+ * Converts from wrapper to thrift schema evolution entry
+ */
@Override
public org.apache.carbondata.format.SchemaEvolutionEntry
- fromWrapperToExternalSchemaEvolutionEntry(SchemaEvolutionEntry wrapperSchemaEvolutionEntry) {
+ fromWrapperToExternalSchemaEvolutionEntry(SchemaEvolutionEntry wrapperSchemaEvolutionEntry) {
org.apache.carbondata.format.SchemaEvolutionEntry thriftSchemaEvolutionEntry =
new org.apache.carbondata.format.SchemaEvolutionEntry(
wrapperSchemaEvolutionEntry.getTimeStamp());
@@ -159,7 +162,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
*/
@Override public org.apache.carbondata.format.ColumnSchema fromWrapperToExternalColumnSchema(
ColumnSchema wrapperColumnSchema) {
-
List<org.apache.carbondata.format.Encoding> encoders =
new ArrayList<org.apache.carbondata.format.Encoding>();
for (Encoding encoder : wrapperColumnSchema.getEncodingList()) {
@@ -179,13 +181,18 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
-
if (wrapperColumnSchema.isSortColumn()) {
Map<String, String> properties = new HashMap<String, String>();
properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
thriftColumnSchema.setColumnProperties(properties);
}
-
+ thriftColumnSchema.setAggregate_function(wrapperColumnSchema.getAggFunction());
+ List<ParentColumnTableRelation> parentColumnTableRelations =
+ wrapperColumnSchema.getParentColumnTableRelations();
+ if (null != parentColumnTableRelations) {
+ thriftColumnSchema.setParentColumnTableRelations(
+ wrapperToThriftRelationList(parentColumnTableRelations));
+ }
return thriftColumnSchema;
}
@@ -271,11 +278,79 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
*/
@Override public org.apache.carbondata.format.TableInfo fromWrapperToExternalTableInfo(
TableInfo wrapperTableInfo, String dbName, String tableName) {
-
org.apache.carbondata.format.TableSchema thriftFactTable =
fromWrapperToExternalTableSchema(wrapperTableInfo.getFactTable());
- return new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache
- .carbondata.format.TableSchema>());
+ org.apache.carbondata.format.TableInfo tableInfo =
+ new org.apache.carbondata.format.TableInfo(thriftFactTable,
+ new ArrayList<org.apache.carbondata.format.TableSchema>());
+ List<DataMapSchema> wrapperChildSchemaList = wrapperTableInfo.getDataMapSchemaList();
+ if (null != wrapperChildSchemaList) {
+ List<org.apache.carbondata.format.DataMapSchema> thriftChildSchemas =
+ fromWrapperToExternalChildSchemaList(wrapperChildSchemaList);
+ tableInfo.setDataMapSchemas(thriftChildSchemas);
+ }
+ return tableInfo;
+ }
+
+ private List<org.apache.carbondata.format.RelationIdentifier> fromWrapperToExternalRI(
+ List<RelationIdentifier> relationIdentifiersList) {
+ List<org.apache.carbondata.format.RelationIdentifier> thriftRelationIdentifierList =
+ new ArrayList<>();
+ for (RelationIdentifier relationIdentifier : relationIdentifiersList) {
+ org.apache.carbondata.format.RelationIdentifier thriftRelationIdentifier =
+ new org.apache.carbondata.format.RelationIdentifier();
+ thriftRelationIdentifier.setDatabaseName(relationIdentifier.getDatabaseName());
+ thriftRelationIdentifier.setTableName(relationIdentifier.getTableName());
+ thriftRelationIdentifier.setTableId(relationIdentifier.getTableId());
+ thriftRelationIdentifierList.add(thriftRelationIdentifier);
+ }
+ return thriftRelationIdentifierList;
+ }
+
+ private List<org.apache.carbondata.format.DataMapSchema> fromWrapperToExternalChildSchemaList(
+ List<DataMapSchema> wrapperChildSchemaList) {
+ List<org.apache.carbondata.format.DataMapSchema> thriftChildSchemas = new ArrayList<>();
+ for (DataMapSchema wrapperChildSchema : wrapperChildSchemaList) {
+ org.apache.carbondata.format.DataMapSchema thriftChildSchema =
+ new org.apache.carbondata.format.DataMapSchema();
+ org.apache.carbondata.format.RelationIdentifier relationIdentifier =
+ new org.apache.carbondata.format.RelationIdentifier();
+ relationIdentifier
+ .setDatabaseName(wrapperChildSchema.getRelationIdentifier().getDatabaseName());
+ relationIdentifier.setTableName(wrapperChildSchema.getRelationIdentifier().getTableName());
+ relationIdentifier.setTableId(wrapperChildSchema.getRelationIdentifier().getTableId());
+ thriftChildSchema.setRelationIdentifire(relationIdentifier);
+ thriftChildSchema.setProperties(wrapperChildSchema.getProperties());
+ thriftChildSchema.setClassName(wrapperChildSchema.getClassName());
+ thriftChildSchema.setChildTableSchema(
+ fromWrapperToExternalTableSchema(wrapperChildSchema.getChildSchema()));
+ thriftChildSchemas.add(thriftChildSchema);
+ }
+ return thriftChildSchemas;
+ }
+
+ private List<org.apache.carbondata.format.ParentColumnTableRelation> wrapperToThriftRelationList(
+ List<ParentColumnTableRelation> wrapperColumnRelations) {
+ List<org.apache.carbondata.format.ParentColumnTableRelation> thriftColumnRelationList =
+ new ArrayList<>();
+
+ for (ParentColumnTableRelation wrapperColumnRealtion : wrapperColumnRelations) {
+ org.apache.carbondata.format.ParentColumnTableRelation thriftColumnTableRelation =
+ new org.apache.carbondata.format.ParentColumnTableRelation();
+ thriftColumnTableRelation.setColumnId(wrapperColumnRealtion.getColumnId());
+ thriftColumnTableRelation.setColumnName(wrapperColumnRealtion.getColumnName());
+ org.apache.carbondata.format.RelationIdentifier thriftRelationIdentifier =
+ new org.apache.carbondata.format.RelationIdentifier();
+ thriftRelationIdentifier
+ .setDatabaseName(wrapperColumnRealtion.getRelationIdentifier().getDatabaseName());
+ thriftRelationIdentifier
+ .setTableName(wrapperColumnRealtion.getRelationIdentifier().getTableName());
+ thriftRelationIdentifier
+ .setTableId(wrapperColumnRealtion.getRelationIdentifier().getTableId());
+ thriftColumnTableRelation.setRelationIdentifier(thriftRelationIdentifier);
+ thriftColumnRelationList.add(thriftColumnTableRelation);
+ }
+ return thriftColumnRelationList;
}
/* (non-Javadoc)
@@ -422,9 +497,17 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setSortColumn(true);
}
}
+ wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function());
+ List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =
+ externalColumnSchema.getParentColumnTableRelations();
+ if (null != parentColumnTableRelation) {
+ wrapperColumnSchema.setParentColumnTableRelations(
+ fromExtrenalToWrapperParentTableColumnRelations(parentColumnTableRelation));
+ }
return wrapperColumnSchema;
}
+
private PartitionType fromExternalToWrapperPartitionType(
org.apache.carbondata.format.PartitionType externalPartitionType) {
if (null == externalPartitionType) {
@@ -494,7 +577,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
org.apache.carbondata.format.BucketingInfo externalBucketInfo) {
List<ColumnSchema> listOfColumns = new ArrayList<ColumnSchema>();
for (org.apache.carbondata.format.ColumnSchema externalColumnSchema :
- externalBucketInfo.table_columns) {
+ externalBucketInfo.table_columns) {
listOfColumns.add(fromExternalToWrapperColumnSchema(externalColumnSchema));
}
return new BucketingInfo(listOfColumns, externalBucketInfo.number_of_buckets);
@@ -517,7 +600,51 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperTableInfo.setStorePath(storePath);
wrapperTableInfo.setFactTable(
fromExternalToWrapperTableSchema(externalTableInfo.getFact_table(), tableName));
+ if (null != externalTableInfo.getDataMapSchemas()) {
+ wrapperTableInfo.setDataMapSchemaList(
+ fromExternalToWrapperChildSchemaList(externalTableInfo.getDataMapSchemas()));
+ }
return wrapperTableInfo;
}
+ @Override public DataMapSchema fromExternalToWrapperDataMapSchema(
+ org.apache.carbondata.format.DataMapSchema thriftDataMapSchema) {
+ RelationIdentifier relationIdentifier =
+ new RelationIdentifier(thriftDataMapSchema.getRelationIdentifire().getDatabaseName(),
+ thriftDataMapSchema.getRelationIdentifire().getTableName(),
+ thriftDataMapSchema.getRelationIdentifire().getTableId());
+ DataMapSchema childSchema = new DataMapSchema(thriftDataMapSchema.getClassName());
+ childSchema.setProperties(thriftDataMapSchema.getProperties());
+ childSchema.setChildSchema(
+ fromExternalToWrapperTableSchema(thriftDataMapSchema.getChildTableSchema(),
+ relationIdentifier.getTableName()));
+ childSchema.setRelationIdentifier(relationIdentifier);
+ return childSchema;
+ }
+
+ private List<ParentColumnTableRelation> fromExtrenalToWrapperParentTableColumnRelations(
+ List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) {
+ List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>();
+ for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation :
+ thirftParentColumnRelation) {
+ RelationIdentifier relationIdentifier =
+ new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(),
+ carbonTableRelation.getRelationIdentifier().getTableName(),
+ carbonTableRelation.getRelationIdentifier().getTableId());
+ ParentColumnTableRelation parentColumnTableRelation =
+ new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(),
+ carbonTableRelation.getColumnName());
+ parentColumnTableRelationList.add(parentColumnTableRelation);
+ }
+ return parentColumnTableRelationList;
+ }
+
+ public List<DataMapSchema> fromExternalToWrapperChildSchemaList(
+ List<org.apache.carbondata.format.DataMapSchema> childSchemaList) {
+ List<DataMapSchema> childSchemas = new ArrayList<>();
+ for (org.apache.carbondata.format.DataMapSchema childSchemaThrift : childSchemaList) {
+ childSchemas.add(fromExternalToWrapperDataMapSchema(childSchemaThrift));
+ }
+ return childSchemas;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index a6738a3..d56a78c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -212,7 +212,7 @@ public class CarbonTable implements Serializable {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
+ columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
allDimensions.add(complexDimension);
dimensionOrdinal =
@@ -226,8 +226,8 @@ public class CarbonTable implements Serializable {
}
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, -1);
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfNoDictSortColumns++;
}
@@ -236,8 +236,8 @@ public class CarbonTable implements Serializable {
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
&& columnSchema.getColumnGroupId() == -1) {
CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else {
@@ -245,15 +245,15 @@ public class CarbonTable implements Serializable {
previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0;
previousColumnGroupId = columnSchema.getColumnGroupId();
CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++,
- columnGroupOrdinal, -1);
+ columnSchema.getSchemaOrdinal(), keyOrdinal++,
+ columnGroupOrdinal, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
}
}
} else {
allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
- columnSchema.getSchemaOrdinal()));
+ columnSchema.getSchemaOrdinal()));
}
}
fillVisibleDimensions(tableSchema.getTableName());
@@ -286,14 +286,14 @@ public class CarbonTable implements Serializable {
*/
private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount,
List<ColumnSchema> listOfColumns, CarbonDimension parentDimension,
- List<CarbonDimension> primitiveDimensions) {
+ List<CarbonDimension> primitiveDimensions) {
for (int i = 0; i < childCount; i++) {
ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ columnSchema.getSchemaOrdinal(), -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -301,8 +301,8 @@ public class CarbonTable implements Serializable {
listOfColumns, complexDimension, primitiveDimensions);
} else {
CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, -1);
parentDimension.getListOfChildDimensions().add(carbonDimension);
primitiveDimensions.add(carbonDimension);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
new file mode 100644
index 0000000..5f92ec8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Child schema class to maintain the child table details inside parent table
+ */
+public class DataMapSchema implements Serializable, Writable {
+
+ private String className;
+
+ private RelationIdentifier relationIdentifier;
+ /**
+ * child table schema
+ */
+ private TableSchema childSchema;
+
+ /**
+ * relation properties
+ */
+ private Map<String, String> properties;
+
+ public DataMapSchema(String className) {
+ this.className = className;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public TableSchema getChildSchema() {
+ return childSchema;
+ }
+
+ public RelationIdentifier getRelationIdentifier() {
+ return relationIdentifier;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setRelationIdentifier(RelationIdentifier relationIdentifier) {
+ this.relationIdentifier = relationIdentifier;
+ }
+
+ public void setChildSchema(TableSchema childSchema) {
+ this.childSchema = childSchema;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeUTF(className);
+ boolean isRelationIdentifierExists = null != relationIdentifier;
+ out.writeBoolean(isRelationIdentifierExists);
+ if (isRelationIdentifierExists) {
+ this.relationIdentifier.write(out);
+ }
+ boolean isChildSchemaExists = null != this.childSchema;
+ out.writeBoolean(isChildSchemaExists);
+ if (isChildSchemaExists) {
+ this.childSchema.write(out);
+ }
+ if (properties == null) {
+ out.writeShort(0);
+ } else {
+ out.writeShort(properties.size());
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ this.className = in.readUTF();
+ boolean isRelationIdnentifierExists = in.readBoolean();
+ if (isRelationIdnentifierExists) {
+ this.relationIdentifier = new RelationIdentifier(null, null, null);
+ this.relationIdentifier.readFields(in);
+ }
+ boolean isChildSchemaExists = in.readBoolean();
+ if (isChildSchemaExists) {
+ this.childSchema = new TableSchema();
+ this.childSchema.readFields(in);
+ }
+
+ int mapSize = in.readShort();
+ this.properties = new HashMap<>(mapSize);
+ for (int i = 0; i < mapSize; i++) {
+ String key = in.readUTF();
+ String value = in.readUTF();
+ this.properties.put(key, value);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
new file mode 100644
index 0000000..9a70b8b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+/**
+ * class to maintain the relation between parent and child
+ */
+public class RelationIdentifier implements Serializable, Writable {
+
+ private String databaseName;
+
+ private String tableName;
+
+ private String tableId;
+
+ public RelationIdentifier(String databaseName, String tableName, String tableId) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.tableId = tableId;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeUTF(databaseName);
+ out.writeUTF(tableName);
+ out.writeUTF(tableId);
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ this.databaseName = in.readUTF();
+ this.tableName = in.readUTF();
+ this.tableId = in.readUTF();
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RelationIdentifier that = (RelationIdentifier) o;
+
+ if (databaseName != null ?
+ !databaseName.equals(that.databaseName) :
+ that.databaseName != null) {
+ return false;
+ }
+ if (tableName != null ? !tableName.equals(that.tableName) : that.tableName != null) {
+ return false;
+ }
+ return tableId != null ? tableId.equals(that.tableId) : that.tableId == null;
+ }
+
+ @Override public int hashCode() {
+ int result = databaseName != null ? databaseName.hashCode() : 0;
+ result = 31 * result + (tableName != null ? tableName.hashCode() : 0);
+ result = 31 * result + (tableId != null ? tableId.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 0c807f6..c236494 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -24,13 +24,15 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
-import java.util.Map;
+import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
/**
* Store the information about the table.
@@ -79,7 +81,12 @@ public class TableInfo implements Serializable, Writable {
// this idenifier is a lazy field which will be created when it is used first time
private AbsoluteTableIdentifier identifier;
+ private List<DataMapSchema> dataMapSchemaList;
+
+ private List<RelationIdentifier> parentRelationIdentifiers;
+
public TableInfo() {
+ dataMapSchemaList = new ArrayList<>();
}
/**
@@ -94,6 +101,23 @@ public class TableInfo implements Serializable, Writable {
*/
public void setFactTable(TableSchema factTable) {
this.factTable = factTable;
+ updateParentRelationIdentifier();
+ }
+
+ private void updateParentRelationIdentifier() {
+ Set<RelationIdentifier> parentRelationIdentifiers = new HashSet<>();
+ this.parentRelationIdentifiers = new ArrayList<>();
+ List<ColumnSchema> listOfColumns = this.factTable.getListOfColumns();
+ for (ColumnSchema columnSchema : listOfColumns) {
+ List<ParentColumnTableRelation> parentColumnTableRelations =
+ columnSchema.getParentColumnTableRelations();
+ if (null != parentColumnTableRelations) {
+ for (int i = 0; i < parentColumnTableRelations.size(); i++) {
+ parentRelationIdentifiers.add(parentColumnTableRelations.get(i).getRelationIdentifier());
+ }
+ }
+ }
+ this.parentRelationIdentifiers.addAll(parentRelationIdentifiers);
}
/**
@@ -160,6 +184,13 @@ public class TableInfo implements Serializable, Writable {
this.storePath = storePath;
}
+ public List<DataMapSchema> getDataMapSchemaList() {
+ return dataMapSchemaList;
+ }
+
+ public void setDataMapSchemaList(List<DataMapSchema> dataMapSchemaList) {
+ this.dataMapSchemaList = dataMapSchemaList;
+ }
/**
* to generate the hash code
*/
@@ -228,10 +259,27 @@ public class TableInfo implements Serializable, Writable {
out.writeLong(lastUpdatedTime);
out.writeUTF(metaDataFilepath);
out.writeUTF(storePath);
+ boolean isChildSchemaExists =
+ null != dataMapSchemaList && dataMapSchemaList.size() > 0 ? true : false;
+ out.writeBoolean(isChildSchemaExists);
+ if (isChildSchemaExists) {
+ out.writeShort(dataMapSchemaList.size());
+ for (int i = 0; i < dataMapSchemaList.size(); i++) {
+ dataMapSchemaList.get(i).write(out);
+ }
+ }
+ boolean isParentTableRelationIndentifierExists =
+ null != parentRelationIdentifiers && parentRelationIdentifiers.size() > 0;
+ out.writeBoolean(isParentTableRelationIndentifierExists);
+ if (isParentTableRelationIndentifierExists) {
+ out.writeShort(parentRelationIdentifiers.size());
+ for (int i = 0; i < parentRelationIdentifiers.size(); i++) {
+ parentRelationIdentifiers.get(i).write(out);
+ }
+ }
}
- @Override
- public void readFields(DataInput in) throws IOException {
+ @Override public void readFields(DataInput in) throws IOException {
this.databaseName = in.readUTF();
this.tableUniqueName = in.readUTF();
this.factTable = new TableSchema();
@@ -239,6 +287,26 @@ public class TableInfo implements Serializable, Writable {
this.lastUpdatedTime = in.readLong();
this.metaDataFilepath = in.readUTF();
this.storePath = in.readUTF();
+ boolean isChildSchemaExists = in.readBoolean();
+ this.dataMapSchemaList = new ArrayList<>();
+ if (isChildSchemaExists) {
+ short numberOfChildTable = in.readShort();
+ for (int i = 0; i < numberOfChildTable; i++) {
+ DataMapSchema childSchema = new DataMapSchema(null);
+ childSchema.readFields(in);
+ dataMapSchemaList.add(childSchema);
+ }
+ }
+ boolean isParentTableRelationIndentifierExists = in.readBoolean();
+ if (isParentTableRelationIndentifierExists) {
+ short parentTableIndentifiersListSize = in.readShort();
+ this.parentRelationIdentifiers = new ArrayList<>();
+ for (int i = 0; i < parentTableIndentifiersListSize; i++) {
+ RelationIdentifier relationIdentifier = new RelationIdentifier(null, null, null);
+ relationIdentifier.readFields(in);
+ this.parentRelationIdentifiers.add(relationIdentifier);
+ }
+ }
}
public AbsoluteTableIdentifier getOrCreateAbsoluteTableIdentifier() {
@@ -262,4 +330,9 @@ public class TableInfo implements Serializable, Writable {
tableInfo.readFields(in);
return tableInfo;
}
+
+ public List<RelationIdentifier> getParentRelationIdentifiers() {
+ return parentRelationIdentifiers;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 7dc41a4..5b013d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -21,6 +21,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -250,4 +251,29 @@ public class TableSchema implements Serializable, Writable {
}
}
+ /**
+ * Below method will be used to build child schema object which will be stored in
+ * parent table
+ *
+ * @param className
+ * @param databaseName
+ * @param queryString
+ * @param queryType
+ *
+ * @return datamap schema
+ */
+ public DataMapSchema buildChildSchema(String className, String databaseName, String queryString,
+ String queryType) {
+ RelationIdentifier relationIdentifier =
+ new RelationIdentifier(databaseName, tableName, tableId);
+ Map<String, String> properties = new HashMap<>();
+ properties.put("CHILD_SELECT QUERY", queryString);
+ properties.put("QUERYTYPE", queryType);
+ DataMapSchema dataMapSchema = new DataMapSchema(className);
+ dataMapSchema.setChildSchema(this);
+ dataMapSchema.setProperties(properties);
+ dataMapSchema.setRelationIdentifier(relationIdentifier);
+ return dataMapSchema;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index bd246a4..4909dd4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -121,6 +121,13 @@ public class ColumnSchema implements Serializable, Writable {
private boolean isSortColumn = false;
/**
+ * aggregate function used in pre aggregate table
+ */
+ private String aggFunction = "";
+
+ private List<ParentColumnTableRelation> parentColumnTableRelations;
+
+ /**
* @return the columnName
*/
public String getColumnName() {
@@ -267,6 +274,15 @@ public class ColumnSchema implements Serializable, Writable {
this.defaultValue = defaultValue;
}
+ public List<ParentColumnTableRelation> getParentColumnTableRelations() {
+ return parentColumnTableRelations;
+ }
+
+ public void setParentColumnTableRelations(
+ List<ParentColumnTableRelation> parentColumnTableRelations) {
+ this.parentColumnTableRelations = parentColumnTableRelations;
+ }
+
/**
* hash code method to check get the hashcode based.
* for generating the hash code only column name and column unique id will considered
@@ -275,7 +291,7 @@ public class ColumnSchema implements Serializable, Writable {
final int prime = 31;
int result = 1;
result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()) +
- ((dataType == null) ? 0 : dataType.hashCode());
+ ((dataType == null) ? 0 : dataType.hashCode());
return result;
}
@@ -408,6 +424,14 @@ public class ColumnSchema implements Serializable, Writable {
isSortColumn = sortColumn;
}
+ public String getAggFunction() {
+ return aggFunction;
+ }
+
+ public void setAggFunction(String aggFunction) {
+ this.aggFunction = aggFunction;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeShort(dataType.getId());
@@ -439,6 +463,16 @@ public class ColumnSchema implements Serializable, Writable {
}
out.writeBoolean(invisible);
out.writeBoolean(isSortColumn);
+ out.writeUTF(null != aggFunction ? aggFunction : "");
+ boolean isParentTableColumnRelationExists =
+ null != parentColumnTableRelations && parentColumnTableRelations.size() > 0;
+ out.writeBoolean(isParentTableColumnRelationExists);
+ if (isParentTableColumnRelationExists) {
+ out.writeShort(parentColumnTableRelations.size());
+ for (int i = 0; i < parentColumnTableRelations.size(); i++) {
+ parentColumnTableRelations.get(i).write(out);
+ }
+ }
}
@Override
@@ -469,5 +503,17 @@ public class ColumnSchema implements Serializable, Writable {
}
this.invisible = in.readBoolean();
this.isSortColumn = in.readBoolean();
+ this.aggFunction = in.readUTF();
+ boolean isParentTableColumnRelationExists = in.readBoolean();
+ if (isParentTableColumnRelationExists) {
+ short parentColumnTableRelationSize = in.readShort();
+ this.parentColumnTableRelations = new ArrayList<>(parentColumnTableRelationSize);
+ for (int i = 0; i < parentColumnTableRelationSize; i++) {
+ ParentColumnTableRelation parentColumnTableRelation =
+ new ParentColumnTableRelation(null, null, null);
+ parentColumnTableRelation.readFields(in);
+ parentColumnTableRelations.add(parentColumnTableRelation);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 15f85d6..35d2e47 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -38,7 +38,9 @@ import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockIndex;
@@ -277,9 +279,35 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setSortColumn(true);
}
}
+ wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function());
+ List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =
+ externalColumnSchema.getParentColumnTableRelations();
+ if (null != parentColumnTableRelation) {
+ wrapperColumnSchema.setParentColumnTableRelations(
+ fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation));
+ }
return wrapperColumnSchema;
}
+ private List<ParentColumnTableRelation> fromThriftToWrapperParentTableColumnRelations(
+ List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) {
+ List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>();
+ for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation :
+ thirftParentColumnRelation) {
+ RelationIdentifier relationIdentifier =
+ new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(),
+ carbonTableRelation.getRelationIdentifier().getTableName(),
+ carbonTableRelation.getRelationIdentifier().getTableId());
+ ParentColumnTableRelation parentColumnTableRelation =
+ new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(),
+ carbonTableRelation.getColumnName());
+ parentColumnTableRelationList.add(parentColumnTableRelation);
+ }
+ return parentColumnTableRelationList;
+ }
+
+
+
/**
* Below method is convert the thrift encoding to wrapper encoding
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/334aa1cc/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 7869378..cee83c1 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -115,6 +115,12 @@ struct ColumnSchema{
* It will have column order which user has provided
*/
16: optional i32 schemaOrdinal
+
+ /**
+ * to maintain the column relation with parent table.
+ * will be usefull in case of pre-aggregate
+ **/
+ 17: optional list<ParentColumnTableRelation> parentColumnTableRelations;
}
/**
@@ -167,7 +173,32 @@ struct TableSchema{
6: optional PartitionInfo partitionInfo; // Partition information
}
+struct RelationIdentifier {
+ 1: optional string databaseName;
+ 2: required string tableName;
+ 3: required string tableId;
+}
+
+struct ParentColumnTableRelation {
+ 1: required RelationIdentifier relationIdentifier;
+ 2: required string columnId;
+ 3: required string columnName
+}
+
+struct DataMapSchema {
+ // class name
+ 1: required string className;
+ // relation indentifier
+ 2: optional RelationIdentifier relationIdentifire;
+ // in case of preaggregate it will be used to maintain the child schema
+ // which will be usefull in case of query and data load
+ 3: optional TableSchema childTableSchema;
+ // to maintain properties like select query, query type like groupby, join
+ 4: optional map<string, string> properties;
+}
+
struct TableInfo{
1: required TableSchema fact_table;
2: required list<TableSchema> aggregate_table_list;
+ 3: optional list<DataMapSchema> dataMapSchemas; // childSchema information
}