You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/03 19:43:23 UTC
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2113]compatibility
fix for v2
[CARBONDATA-2113]compatibility fix for v2
Fixes related to backword compatibility:
Count() issue:
when count() is run on old store where file format version is V2, it was unable to get the files, because when forming the filepath while creating the table block info object from index file info, the path was formed with double slash(//), beacuse in v2, local path was stored. so when trying to get this path with key, it was failing. So form the correct file path.
Select * issue:
when select * is run, then only the datachunck was considered as whole data and while uncompressing the measure data, it was failing. So, read proper data and datachunck before uncompressing.
Read Metadata file:
when readMetadataFile is called , the while reading the schema, it was explicitly returning null, so columns will be null and hence some nullpointerxception was coming. so do not return null, return proper schema, by reading the footer.
Vesion compatibility:
calculation the number of pages to be filled based on row count was not handled for V2 version, handled same, as no. of rows per page is different in V2 and V3
This closes #1901
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1202e209
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1202e209
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1202e209
Branch: refs/heads/branch-1.3
Commit: 1202e209eccda172f9671fa0cc2deeebeb4af456
Parents: 1248bd4
Author: akashrn5 <ak...@gmail.com>
Authored: Thu Feb 1 12:42:49 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Feb 1 22:16:39 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonVersionConstants.java | 5 +++++
.../v2/CompressedMeasureChunkFileBasedReaderV2.java | 15 ++++++++++-----
.../blockletindex/BlockletDataRefNodeWrapper.java | 13 +++++++++++--
.../carbondata/core/mutate/CarbonUpdateUtil.java | 10 ++++------
.../core/util/AbstractDataFileFooterConverter.java | 3 ++-
.../core/util/DataFileFooterConverter2.java | 2 +-
6 files changed, 33 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
index 2d58b0b..22fbaf2 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
@@ -49,6 +49,11 @@ public final class CarbonVersionConstants {
*/
public static final String CARBONDATA_BUILD_DATE;
+ /**
+ * number of rows per blocklet column page default value for V2 version
+ */
+ public static final int NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2 = 120000;
+
static {
// create input stream for CARBONDATA_VERSION_INFO_FILE
InputStream resourceStream = Thread.currentThread().getContextClassLoader()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 2ddc202..d61f98a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -52,7 +52,14 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
throws IOException {
int dataLength = 0;
if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
- dataLength = measureColumnChunkLength.get(columnIndex);
+ DataChunk2 metadataChunk = null;
+ synchronized (fileReader) {
+ metadataChunk = CarbonUtil.readDataChunk(ByteBuffer.wrap(fileReader
+ .readByteArray(filePath, measureColumnChunkOffsets.get(columnIndex),
+ measureColumnChunkLength.get(columnIndex))), 0,
+ measureColumnChunkLength.get(columnIndex));
+ }
+ dataLength = measureColumnChunkLength.get(columnIndex) + metadataChunk.data_page_length;
} else {
long currentMeasureOffset = measureColumnChunkOffsets.get(columnIndex);
dataLength = (int) (measureColumnChunkOffsets.get(columnIndex + 1) - currentMeasureOffset);
@@ -115,9 +122,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
ByteBuffer rawData = measureRawColumnChunk.getRawData();
DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint,
measureColumnChunkLength.get(blockIndex));
- if (measureColumnChunkOffsets.size() - 1 != blockIndex) {
- copyPoint += measureColumnChunkLength.get(blockIndex);
- }
+ copyPoint += measureColumnChunkLength.get(blockIndex);
ColumnPage page = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint);
page.setNullBits(getNullBitSet(measureColumnChunk.presence));
@@ -130,7 +135,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
byte[] encodedMeta = encoder_meta.get(0).array();
- ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
+ ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta);
ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
byte[] rawData = measureRawColumnChunk.getRawData().array();
return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index b672c58..4e49ede 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.constants.CarbonVersionConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -56,8 +57,16 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
int[] pageRowCount = new int[detailInfo.getPagesCount()];
- int numberOfPagesCompletelyFilled = detailInfo.getRowCount()
- / CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ int numberOfPagesCompletelyFilled = detailInfo.getRowCount();
+ // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the number
+ // of pages filled
+ if (blockInfo.getVersion() == ColumnarFormatVersion.V2) {
+ numberOfPagesCompletelyFilled /=
+ CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
+ } else {
+ numberOfPagesCompletelyFilled /=
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
int lastPageRowCount = detailInfo.getRowCount()
% CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 0e4eec7..c5f61c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -587,26 +587,24 @@ public class CarbonUpdateUtil {
*/
private static void deleteStaleCarbonDataFiles(LoadMetadataDetails segment,
CarbonFile[] allSegmentFiles, SegmentUpdateStatusManager updateStatusManager) {
- boolean doForceDelete = true;
- boolean isAbortedFile = true;
CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager
.getUpdateDeltaFilesList(segment.getLoadName(), false,
CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles,
- isAbortedFile);
+ true);
// now for each invalid delta file need to check the query execution time out
// and then delete.
for (CarbonFile invalidFile : invalidUpdateDeltaFiles) {
- compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+ compareTimestampsAndDelete(invalidFile, true, false);
}
// do the same for the index files.
CarbonFile[] invalidIndexFiles = updateStatusManager
.getUpdateDeltaFilesList(segment.getLoadName(), false,
CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles,
- isAbortedFile);
+ true);
// now for each invalid index file need to check the query execution time out
// and then delete.
for (CarbonFile invalidFile : invalidIndexFiles) {
- compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+ compareTimestampsAndDelete(invalidFile, true, false);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index c7bc6aa..94a041a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -208,7 +208,8 @@ public abstract class AbstractDataFileFooterConverter {
if (fileName.lastIndexOf("/") > 0) {
fileName = fileName.substring(fileName.lastIndexOf("/"));
}
- tableBlockInfo.setFilePath(parentPath + "/" + fileName);
+ fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/");
+ tableBlockInfo.setFilePath(parentPath + fileName);
return tableBlockInfo;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 8cd437f..863e1df 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -141,6 +141,6 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter {
}
@Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
- return null;
+ return new DataFileFooterConverter().getSchema(tableBlockInfo);
}
}