You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:12:53 UTC
[29/50] [abbrv] incubator-carbondata git commit: Use map to store
SegmentProperties and reuse SegmentProperties.
Use map to store SegmentProperties and reuse SegmentProperties.
Add comment.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/33a3858a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/33a3858a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/33a3858a
Branch: refs/heads/12-dev
Commit: 33a3858a8c28551dd37f4a530cb52d97d888f810
Parents: 58be73f
Author: Yadong Qi <qi...@gmail.com>
Authored: Fri Apr 21 16:21:04 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 22:15:44 2017 -0700
----------------------------------------------------------------------
.../core/datastore/SegmentTaskIndexStore.java | 61 +++++++++++++++++++-
.../core/datastore/block/SegmentTaskIndex.java | 10 ++--
.../schema/table/column/ColumnSchema.java | 10 +++-
.../datastore/block/SegmentTaskIndexTest.java | 4 +-
4 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 862455e..b1eaa84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -31,12 +32,14 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CarbonLRUCache;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,6 +71,9 @@ public class SegmentTaskIndexStore
*/
private Map<String, Object> segmentLockMap;
+ private Map<SegmentPropertiesWrapper, SegmentProperties> segmentProperties =
+ new HashMap<SegmentPropertiesWrapper, SegmentProperties>();
+
/**
* constructor to initialize the SegmentTaskIndexStore
*
@@ -345,7 +351,26 @@ public class SegmentTaskIndexStore
List<DataFileFooter> footerList = CarbonUtil
.readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber,
tableBlockInfoList, tableIdentifier);
- AbstractIndex segment = new SegmentTaskIndex();
+
+ // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality are
+ // the same.
+ List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
+ int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
+ SegmentPropertiesWrapper segmentPropertiesWrapper =
+ new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
+ SegmentProperties segmentProperties;
+ if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
+ segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
+ } else {
+ // create a metadata details
+ // this will be useful in query handling
+ // all the data file metadata will have common segment properties we
+ // can use first one to get create the segment properties
+ segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
+ this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
+ }
+
+ AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
// file path of only first block is passed as it all table block info path of
// same task id will be same
segment.buildIndex(footerList);
@@ -396,4 +421,38 @@ public class SegmentTaskIndexStore
return result;
}
}
+
+ /**
+ * This class wraps tableIdentifier, columnsInTable and columnCardinality as a key to determine
+ * whether the SegmentProperties object can be reused.
+ */
+ public static class SegmentPropertiesWrapper {
+ private AbsoluteTableIdentifier tableIdentifier;
+ private List<ColumnSchema> columnsInTable;
+ private int[] columnCardinality;
+
+ public SegmentPropertiesWrapper(AbsoluteTableIdentifier tableIdentifier,
+ List<ColumnSchema> columnsInTable, int[] columnCardinality) {
+ this.tableIdentifier = tableIdentifier;
+ this.columnsInTable = columnsInTable;
+ this.columnCardinality = columnCardinality;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SegmentPropertiesWrapper)) {
+ return false;
+ }
+ SegmentPropertiesWrapper other = (SegmentPropertiesWrapper) obj;
+ return tableIdentifier.equals(other.tableIdentifier)
+ && columnsInTable.equals(other.columnsInTable)
+ && Arrays.equals(columnCardinality, other.columnCardinality);
+ }
+
+ @Override
+ public int hashCode() {
+ return tableIdentifier.hashCode()
+ + columnsInTable.hashCode() + Arrays.hashCode(columnCardinality);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
index 7293323..f577ede 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
@@ -29,17 +29,15 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
*/
public class SegmentTaskIndex extends AbstractIndex {
+ public SegmentTaskIndex(SegmentProperties segmentProperties) {
+ this.segmentProperties = segmentProperties;
+ }
+
/**
* Below method is store the blocks in some data structure
*
*/
public void buildIndex(List<DataFileFooter> footerList) {
- // create a metadata details
- // this will be useful in query handling
- // all the data file metadata will have common segment properties we
- // can use first one to get create the segment properties
- segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(),
- footerList.get(0).getSegmentInfo().getColumnCardinality());
// create a segment builder info
// in case of segment create we do not need any file path and each column value size
// as Btree will be build as per min max and start key
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 7aa0900..abe065c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -263,7 +263,8 @@ public class ColumnSchema implements Serializable {
@Override public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((columnName == null) ? 0 : columnName.hashCode());
+ result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()) +
+ ((dataType == null) ? 0 : dataType.hashCode());
return result;
}
@@ -288,6 +289,13 @@ public class ColumnSchema implements Serializable {
} else if (!columnName.equals(other.columnName)) {
return false;
}
+ if (dataType == null) {
+ if (other.dataType != null) {
+ return false;
+ }
+ } else if (!dataType.equals(other.dataType)) {
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
index ac2cfaa..4c08c6b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
@@ -58,7 +58,6 @@ public class SegmentTaskIndexTest {
@Mock public void build(BTreeBuilderInfo segmentBuilderInfos) {}
};
long numberOfRows = 100;
- SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex();
columnSchema.setColumnName("employeeName");
columnSchemaList.add(new ColumnSchema());
@@ -68,6 +67,9 @@ public class SegmentTaskIndexTest {
footer.setNumberOfRows(numberOfRows);
footerList.add(footer);
+ SegmentProperties properties = new SegmentProperties(footerList.get(0).getColumnInTable(),
+ footerList.get(0).getSegmentInfo().getColumnCardinality());
+ SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex(properties);
segmentTaskIndex.buildIndex(footerList);
assertEquals(footerList.get(0).getNumberOfRows(), numberOfRows);