You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/06 10:00:31 UTC
[14/20] incubator-carbondata git commit: [CARBONDATA-139] Sortindex
read by by appending offset from metadata (#905)
[CARBONDATA-139] Sortindex read by by appending offset from metadata (#905)
During sort index reading phase, read offset from dictionary chunk meta and then get respective file.
Corrected testcase
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1721d40d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1721d40d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1721d40d
Branch: refs/heads/master
Commit: 1721d40d3d44d91c8f1115febc8a449071129f22
Parents: 07fe4d2
Author: ashokblend <as...@gmail.com>
Authored: Thu Aug 4 16:54:07 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Thu Aug 4 16:54:07 2016 +0530
----------------------------------------------------------------------
.../CarbonDictionarySortIndexReaderImpl.java | 43 ++++++++++++++++----
...CarbonDictionarySortIndexReaderImplTest.java | 15 ++++++-
...CarbonDictionarySortIndexWriterImplTest.java | 15 +++++++
3 files changed, 65 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1721d40d/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 54e8950..3ec6e7e 100644
--- a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -28,9 +28,11 @@ import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.carbon.ColumnIdentifier;
import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
+import org.carbondata.core.reader.CarbonDictionaryMetadataReader;
+import org.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.carbondata.core.reader.ThriftReader;
import org.carbondata.core.service.PathService;
-import org.carbondata.core.util.CarbonUtil;
import org.carbondata.format.ColumnSortInfo;
import org.apache.thrift.TBase;
@@ -152,13 +154,14 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort
protected void initPath() {
PathService pathService = CarbonCommonFactory.getPathService();
- CarbonTablePath carbonTablePath = pathService
- .getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
- String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
- long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
- this.sortIndexFilePath =
- carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset);
+ CarbonTablePath carbonTablePath =
+ pathService.getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
try {
+ CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry =
+ getChunkMetaObjectForLastSegmentEntry();
+ long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset();
+ this.sortIndexFilePath =
+ carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset);
if (!FileFactory
.isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) {
this.sortIndexFilePath =
@@ -171,6 +174,32 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort
}
/**
+ * This method will read the dictionary chunk metadata thrift object for last entry
+ *
+ * @return last entry of dictionary meta chunk
+ * @throws IOException if an I/O error occurs
+ */
+ private CarbonDictionaryColumnMetaChunk getChunkMetaObjectForLastSegmentEntry()
+ throws IOException {
+ CarbonDictionaryMetadataReader columnMetadataReaderImpl = getDictionaryMetadataReader();
+ try {
+ // read the last segment entry for dictionary metadata
+ return columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
+ } finally {
+ // Close metadata reader
+ columnMetadataReaderImpl.close();
+ }
+ }
+
+ /**
+ * @return
+ */
+ protected CarbonDictionaryMetadataReader getDictionaryMetadataReader() {
+ return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier,
+ columnIdentifier);
+ }
+
+ /**
* This method will open the dictionary sort index file stream for reading
*
* @throws IOException in case any I/O errors occurs
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1721d40d/core/src/test/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
index 8b10c32..3accaa0 100644
--- a/core/src/test/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
+++ b/core/src/test/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
@@ -18,6 +18,7 @@
*/
package org.carbondata.core.reader.sortindex;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -27,6 +28,9 @@ import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.carbon.ColumnIdentifier;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.writer.CarbonDictionaryWriter;
+import org.carbondata.core.writer.CarbonDictionaryWriterImpl;
import org.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
import org.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
import org.apache.commons.lang.ArrayUtils;
@@ -60,10 +64,19 @@ public class CarbonDictionarySortIndexReaderImplTest {
CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon",
UUID.randomUUID().toString());
ColumnIdentifier columnIdentifier = new ColumnIdentifier("Name", null, null);
+ CarbonDictionaryWriter dictionaryWriter = new CarbonDictionaryWriterImpl(hdfsStorePath,
+ carbonTableIdentifier, columnIdentifier);
+ String metaFolderPath =hdfsStorePath+File.separator+carbonTableIdentifier.getDatabaseName()+File.separator+carbonTableIdentifier.getTableName()+File.separator+"Metadata";
+ CarbonUtil.checkAndCreateFolder(metaFolderPath);
CarbonDictionarySortIndexWriter dictionarySortIndexWriter =
new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, hdfsStorePath);
List<int[]> expectedData = prepareExpectedData();
-
+ int[] data = expectedData.get(0);
+ for(int i=0;i<data.length;i++) {
+ dictionaryWriter.write(String.valueOf(data[i]));
+ }
+ dictionaryWriter.close();
+ dictionaryWriter.commit();
List<Integer> sortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(0)));
List<Integer> invertedSortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(1)));
dictionarySortIndexWriter.writeSortIndex(sortIndex);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1721d40d/core/src/test/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java b/core/src/test/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
index b814fa8..7a28da5 100644
--- a/core/src/test/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
+++ b/core/src/test/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
@@ -18,6 +18,7 @@
*/
package org.carbondata.core.writer.sortindex;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -29,6 +30,9 @@ import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.impl.FileFactory;
import org.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.writer.CarbonDictionaryWriter;
+import org.carbondata.core.writer.CarbonDictionaryWriterImpl;
import org.apache.commons.lang.ArrayUtils;
import org.junit.After;
import org.junit.Assert;
@@ -62,9 +66,20 @@ public class CarbonDictionarySortIndexWriterImplTest {
CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon", UUID.randomUUID().toString());
ColumnIdentifier columnIdentifier = new ColumnIdentifier("Name", null, null);
+ String metaFolderPath =hdfsStorePath+File.separator+carbonTableIdentifier.getDatabaseName()+File.separator+carbonTableIdentifier.getTableName()+File.separator+"Metadata";
+ CarbonUtil.checkAndCreateFolder(metaFolderPath);
+ CarbonDictionaryWriter dictionaryWriter = new CarbonDictionaryWriterImpl(hdfsStorePath,
+ carbonTableIdentifier, columnIdentifier);
CarbonDictionarySortIndexWriter dictionarySortIndexWriter =
new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, storePath);
List<int[]> indexList = prepareExpectedData();
+ int[] data = indexList.get(0);
+ for(int i=0;i<data.length;i++) {
+ dictionaryWriter.write(String.valueOf(data[i]));
+ }
+ dictionaryWriter.close();
+ dictionaryWriter.commit();
+
List<Integer> sortIndex = Arrays.asList(ArrayUtils.toObject(indexList.get(0)));
List<Integer> invertedSortIndex = Arrays.asList(ArrayUtils.toObject(indexList.get(1)));
dictionarySortIndexWriter.writeSortIndex(sortIndex);