You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/02/27 15:49:26 UTC
[1/2] incubator-carbondata git commit: Save useInvertedIndex info
into thrift store
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 76c059a29 -> 1474bbf9a
Save useInvertedIndex info into thrift store
Save useInvertedIndex info into thrift store
Fix the judge of no_dic_col
add commont
style
Add comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/96a75b30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/96a75b30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/96a75b30
Branch: refs/heads/master
Commit: 96a75b30ae327475d22db152eaed7853dca9799c
Parents: 76c059a
Author: Zhangshunyu <zh...@huawei.com>
Authored: Thu Sep 8 15:48:03 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Feb 27 23:48:29 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 ++
.../metadata/schema/table/column/ColumnSchema.java | 17 ++++++++++-------
.../carbondata/core/util/CarbonMetadataUtil.java | 4 +++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 ++--
.../csvbased/CarbonCSVBasedSeqGenStep.java | 6 +++++-
5 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 146b78e..9e890ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -804,6 +804,8 @@ public final class CarbonCommonConstants {
public static final String COLUMN_PROPERTIES = "columnproperties";
// table block size in MB
public static final String TABLE_BLOCKSIZE = "table_blocksize";
+ // set in column level to disable inverted index
+ public static final String NO_INVERTED_INDEX = "no_inverted_index";
/**
* this variable is to enable/disable identify high cardinality during first data loading
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 54114d3..7aa0900 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -73,11 +73,6 @@ public class ColumnSchema implements Serializable {
private boolean isDimensionColumn;
/**
- * Whether the column should use inverted index
- */
- private boolean useInvertedIndex = true;
-
- /**
* The group ID for column used for row format columns,
* where in columns in each group are chunked together.
*/
@@ -174,14 +169,22 @@ public class ColumnSchema implements Serializable {
* the isUseInvertedIndex
*/
public boolean isUseInvertedIndex() {
- return useInvertedIndex;
+ return this.hasEncoding(Encoding.INVERTED_INDEX);
}
/**
* @param useInvertedIndex the useInvertedIndex to set
*/
public void setUseInvertedIndex(boolean useInvertedIndex) {
- this.useInvertedIndex = useInvertedIndex;
+ if (useInvertedIndex) {
+ if (!hasEncoding(Encoding.INVERTED_INDEX)) {
+ this.getEncodingList().add(Encoding.INVERTED_INDEX);
+ }
+ } else {
+ if (hasEncoding(Encoding.INVERTED_INDEX)) {
+ this.getEncodingList().remove(Encoding.INVERTED_INDEX);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 55c0302..371855b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -345,7 +345,9 @@ public class CarbonMetadataUtil {
if (!isSortedKeyColumn[i]) {
dataChunk.setRowid_page_offset(blockletInfoColumnar.getKeyBlockIndexOffSets()[j]);
dataChunk.setRowid_page_length(blockletInfoColumnar.getKeyBlockIndexLength()[j]);
- encodings.add(Encoding.INVERTED_INDEX);
+ if (!encodings.contains(Encoding.INVERTED_INDEX)) {
+ encodings.add(Encoding.INVERTED_INDEX);
+ }
j++;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 742a145..9bfa8a9 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -448,9 +448,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
var noInvertedIdxColsProps: Array[String] = Array[String]()
var noInvertedIdxCols: Seq[String] = Seq[String]()
- if (tableProperties.get("NO_INVERTED_INDEX").isDefined) {
+ if (tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).isDefined) {
noInvertedIdxColsProps =
- tableProperties.get("NO_INVERTED_INDEX").get.split(',').map(_.trim)
+ tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).get.split(',').map(_.trim)
noInvertedIdxColsProps.foreach { noInvertedIdxColProp =>
if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 6e0a4a4..3edd43c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -51,6 +51,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -1933,7 +1934,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
dimListExcludingNoDictionaryColumn =
new ArrayList<>(dimensionsList.size() - meta.noDictionaryCols.length);
for (CarbonDimension dimension : dimensionsList) {
- if (!dimension.getEncoder().isEmpty()) {
+ // Here if dimension.getEncoder() lnly contains Encoding.INVERTED_INDEX, it
+ // means that NoDicColumn using InvertedIndex, so not put it into dic dims list.
+ if (!dimension.getEncoder().isEmpty() && !((1 == dimension.getEncoder().size()) &&
+ dimension.getEncoder().contains(Encoding.INVERTED_INDEX))) {
dimListExcludingNoDictionaryColumn.add(dimension);
}
}
[2/2] incubator-carbondata git commit: Fix the bug of inverted index
that store inverted index in metadata by using Encoding.INVERTED_INDEX. This
closes #603
Posted by ja...@apache.org.
Fix the bug of inverted index that store inverted index in metadata by using Encoding.INVERTED_INDEX. This closes #603
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1474bbf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1474bbf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1474bbf9
Branch: refs/heads/master
Commit: 1474bbf9a78498c4e1fc157a3ce58eaf36df8173
Parents: 76c059a 96a75b3
Author: jackylk <ja...@huawei.com>
Authored: Mon Feb 27 23:49:02 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Feb 27 23:49:02 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 ++
.../metadata/schema/table/column/ColumnSchema.java | 17 ++++++++++-------
.../carbondata/core/util/CarbonMetadataUtil.java | 4 +++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 ++--
.../csvbased/CarbonCSVBasedSeqGenStep.java | 6 +++++-
5 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------