You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/11/13 19:21:13 UTC
[8/9] carbondata git commit: [CARBONDATA-1520] [PreAgg] Support
pre-aggregate table load
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cc0e6f1e/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 5dbd177..22d15d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -25,11 +25,15 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.dictionary.client.DictionaryClient;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
@@ -76,12 +80,40 @@ public class FieldEncoderFactory {
isEmptyBadRecord);
} else if (dataField.getColumn().hasEncoding(Encoding.DICTIONARY) &&
!dataField.getColumn().isComplex()) {
- return new DictionaryFieldConverterImpl(dataField, cache, carbonTableIdentifier, nullFormat,
- index, client, useOnePass, storePath, localCache, isEmptyBadRecord);
+ DictionaryColumnUniqueIdentifier identifier = null;
+ // if parent column table relation is not null then it's a child table
+ // in case of child table it will use parent table dictionary
+ if (null == dataField.getColumn().getColumnSchema().getParentColumnTableRelations()
+ || dataField.getColumn().getColumnSchema().getParentColumnTableRelations().isEmpty()) {
+ identifier = new DictionaryColumnUniqueIdentifier(carbonTableIdentifier,
+ dataField.getColumn().getColumnIdentifier(), dataField.getColumn().getDataType(),
+ CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier));
+ return new DictionaryFieldConverterImpl(dataField, cache, carbonTableIdentifier,
+ nullFormat, index, client, useOnePass, storePath, localCache, isEmptyBadRecord,
+ identifier);
+ } else {
+ ParentColumnTableRelation parentColumnTableRelation =
+ dataField.getColumn().getColumnSchema().getParentColumnTableRelations().get(0);
+ RelationIdentifier relationIdentifier =
+ parentColumnTableRelation
+ .getRelationIdentifier();
+ CarbonTableIdentifier parentTableIdentifier =
+ new CarbonTableIdentifier(relationIdentifier.getDatabaseName(),
+ relationIdentifier.getTableName(), relationIdentifier.getTableId());
+ ColumnIdentifier parentColumnIdentifier =
+ new ColumnIdentifier(parentColumnTableRelation.getColumnId(), null,
+ dataField.getColumn().getDataType());
+ identifier =
+ new DictionaryColumnUniqueIdentifier(parentTableIdentifier, parentColumnIdentifier,
+ dataField.getColumn().getDataType(),
+ CarbonStorePath.getCarbonTablePath(storePath, parentTableIdentifier));
+ return new DictionaryFieldConverterImpl(dataField, cache, parentTableIdentifier,
+ nullFormat, index, null, false, storePath, null, isEmptyBadRecord, identifier);
+ }
} else if (dataField.getColumn().isComplex()) {
return new ComplexFieldConverterImpl(
createComplexType(dataField, cache, carbonTableIdentifier,
- client, useOnePass, storePath, localCache), index);
+ client, useOnePass, storePath, localCache), index);
} else {
return new NonDictionaryFieldConverterImpl(dataField, nullFormat, index, isEmptyBadRecord);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cc0e6f1e/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 6a156a6..acd7fed 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -163,6 +163,16 @@ public class CarbonLoadModel implements Serializable {
*/
private String globalSortPartitions;
+ private boolean isAggLoadRequest;
+
+ public boolean isAggLoadRequest() {
+ return isAggLoadRequest;
+ }
+
+ public void setAggLoadRequest(boolean aggLoadRequest) {
+ isAggLoadRequest = aggLoadRequest;
+ }
+
/**
* get escape char
*
@@ -353,6 +363,7 @@ public class CarbonLoadModel implements Serializable {
copy.sortScope = sortScope;
copy.batchSortSizeInMb = batchSortSizeInMb;
copy.badRecordsLocation = badRecordsLocation;
+ copy.isAggLoadRequest = isAggLoadRequest;
return copy;
}
@@ -399,6 +410,7 @@ public class CarbonLoadModel implements Serializable {
copy.isEmptyDataBadRecord = isEmptyDataBadRecord;
copy.sortScope = sortScope;
copy.batchSortSizeInMb = batchSortSizeInMb;
+ copy.isAggLoadRequest = isAggLoadRequest;
return copy;
}
@@ -449,6 +461,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.sortScope = sortScope;
copyObj.batchSortSizeInMb = batchSortSizeInMb;
copyObj.badRecordsLocation = badRecordsLocation;
+ copyObj.isAggLoadRequest = isAggLoadRequest;
return copyObj;
}