You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/08/07 13:10:02 UTC
[38/50] [abbrv] carbondata git commit: [CARBONDATA-2804] fix the bug
when bloom filter or preaggregate datamap tried to be created on older V1-V2
version stores
[CARBONDATA-2804] fix the bug when bloom filter or preaggregate datamap tried to be created on older V1-V2 version stores
This PR change read carbon file version from carbondata file header to
carbonindex file header, because the version filed of carondata file
header is not compatible with older v1/v2 version.
This closes #2601
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9336924d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9336924d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9336924d
Branch: refs/heads/external-format
Commit: 9336924de3f3103507335b5ffbf9c0145be4812f
Parents: 26d9f3d
Author: ndwangsen <lu...@huawei.com>
Authored: Thu Aug 2 16:21:22 2018 +0800
Committer: xuchuanyin <xu...@hust.edu.cn>
Committed: Fri Aug 3 15:25:46 2018 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 85 ++++++++++++++------
1 file changed, 59 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9336924d/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 3eb6aae..205d160 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator;
import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
@@ -88,7 +89,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockletHeader;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
-import org.apache.carbondata.format.FileHeader;
+import org.apache.carbondata.format.IndexHeader;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -3194,11 +3195,33 @@ public final class CarbonUtil {
*/
public static ColumnarFormatVersion getFormatVersion(CarbonTable carbonTable)
throws IOException {
- String storePath = null;
- // if the carbontable is support flat folder
+ String segmentPath = null;
boolean supportFlatFolder = carbonTable.isSupportFlatFolder();
+ CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
+ ColumnarFormatVersion version = null;
+ SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
+ CarbonProperties carbonProperties = CarbonProperties.getInstance();
+ // if the carbontable is support flat folder
if (supportFlatFolder) {
- storePath = carbonTable.getTablePath();
+ segmentPath = carbonTable.getTablePath();
+ FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
+ if (FileFactory.isFileExist(segmentPath, fileType)) {
+ fileStore.readAllIIndexOfSegment(segmentPath);
+ Map<String, byte[]> carbonIndexMap = fileStore.getCarbonIndexMap();
+ if (carbonIndexMap.size() == 0) {
+ version = carbonProperties.getFormatVersion();
+ }
+ for (byte[] fileData : carbonIndexMap.values()) {
+ try {
+ indexReader.openThriftReader(fileData);
+ IndexHeader indexHeader = indexReader.readIndexHeader();
+ version = ColumnarFormatVersion.valueOf((short)indexHeader.getVersion());
+ break;
+ } finally {
+ indexReader.closeThriftReader();
+ }
+ }
+ }
} else {
// get the valid segments
SegmentStatusManager segmentStatusManager =
@@ -3206,34 +3229,44 @@ public final class CarbonUtil {
SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo =
segmentStatusManager.getValidAndInvalidSegments();
List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments();
- CarbonProperties carbonProperties = CarbonProperties.getInstance();
if (validSegments.isEmpty()) {
return carbonProperties.getFormatVersion();
}
- storePath = carbonTable.getSegmentPath(validSegments.get(0).getSegmentNo());
- }
-
- CarbonFile[] carbonFiles = FileFactory
- .getCarbonFile(storePath)
- .listFiles(new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- if (file == null) {
- return false;
+ // get the carbon index file header from a valid segment
+ for (Segment segment : validSegments) {
+ segmentPath = carbonTable.getSegmentPath(segment.getSegmentNo());
+ FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
+ if (FileFactory.isFileExist(segmentPath, fileType)) {
+ fileStore.readAllIIndexOfSegment(segmentPath);
+ Map<String, byte[]> carbonIndexMap = fileStore.getCarbonIndexMap();
+ if (carbonIndexMap.size() == 0) {
+ LOGGER.warn("the valid segment path: " + segmentPath +
+ " does not exist in the system of table: " + carbonTable.getTableUniqueName());
+ continue;
+ }
+ for (byte[] fileData : carbonIndexMap.values()) {
+ try {
+ indexReader.openThriftReader(fileData);
+ IndexHeader indexHeader = indexReader.readIndexHeader();
+ version = ColumnarFormatVersion.valueOf((short)indexHeader.getVersion());
+ break;
+ } finally {
+ indexReader.closeThriftReader();
}
- return file.getName().endsWith("carbondata");
}
- });
- if (carbonFiles == null || carbonFiles.length < 1) {
- return CarbonProperties.getInstance().getFormatVersion();
+ // if get the carbon file version from a valid segment, then end
+ if (version != null) {
+ break;
+ }
+ }
+ }
+ // if all valid segments path does not in the system,
+ // then the carbon file verion as default
+ if (version == null) {
+ version = CarbonProperties.getInstance().getFormatVersion();
+ }
}
-
- CarbonFile carbonFile = carbonFiles[0];
- // get the carbon file header
- CarbonHeaderReader headerReader = new CarbonHeaderReader(carbonFile.getCanonicalPath());
- FileHeader fileHeader = headerReader.readHeader();
- int version = fileHeader.getVersion();
- return ColumnarFormatVersion.valueOf((short)version);
+ return version;
}
/**