You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:12:26 UTC
[02/50] [abbrv] incubator-carbondata git commit: Improve Carbon index
file loading file in case of big cluster
Improve Carbon index file loading file in case of big cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8fc9c7bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8fc9c7bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8fc9c7bf
Branch: refs/heads/12-dev
Commit: 8fc9c7bf67c1e5678cfa955266dfe005fed4a5d5
Parents: 174bfad
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Apr 21 20:42:10 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Apr 22 00:30:32 2017 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 8 +++--
.../core/util/path/CarbonTablePath.java | 33 ++++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 7d57fb8..8956549 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -990,7 +990,9 @@ public final class CarbonUtil {
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
- bucketNumber);
+ bucketNumber, CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+ tableBlockInfoList.get(0).getVersion());
CarbonFile carbonFile = FileFactory
.getCarbonFile(carbonIndexFilePath, FileFactory.getFileType(carbonIndexFilePath));
// in case of carbonIndex file whole file is meta only so reading complete file.
@@ -1232,7 +1234,9 @@ public final class CarbonUtil {
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
- bucketNumber);
+ bucketNumber, CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+ tableBlockInfoList.get(0).getVersion());
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
// read the index info and return
return fileFooterConverter.getIndexInfo(carbonIndexFilePath, tableBlockInfoList);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 3e81fbb..918772a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -23,6 +23,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.hadoop.fs.Path;
@@ -274,6 +275,38 @@ public class CarbonTablePath extends Path {
+ "]");
}
}
+
+ /**
+ * Below method will be used to get the carbon index file path
+ * @param taskId
+ * task id
+ * @param partitionId
+ * partition id
+ * @param segmentId
+ * segment id
+ * @param bucketNumber
+ * bucket number
+ * @param timeStamp
+ * timestamp
+ * @return carbon index file path
+ */
+ public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
+ String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ switch (columnarFormatVersion) {
+ case V1:
+ case V2:
+ return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ default:
+ String segmentDir = getSegmentDir(partitionId, segmentId);
+ return segmentDir + File.separator + getCarbonIndexFileName(taskId,
+ Integer.parseInt(bucketNumber), timeStamp);
+ }
+ }
+
+ private static String getCarbonIndexFileName(String taskNo, int bucketNumber,
+ String factUpdatedtimeStamp) {
+ return taskNo + "-" + bucketNumber + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
+ }
/**
* Below method will be used to get the index file present in the segment folder
* based on task id