You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/06/05 06:18:09 UTC
carbondata git commit: [CARBONDATA-2571] Calculating the carbonindex
and carbondata file size of a table is wrong
Repository: carbondata
Updated Branches:
refs/heads/master 92d9b9256 -> 27d705998
[CARBONDATA-2571] Calculating the carbonindex and carbondata file size of a table is wrong
Problem:
While calculating the carbonindex files size, we are checking either index file or merge file. But in PR#2333, implementation is changed to fill both
the file name and the merge file name. So, we have to consider both fields.
Solution:
While calculating the carbonindex files size, we have to consider both the files and mergeFileName fields. We should get the list of index files from
these 2 fields and then calculate the size of the files.
This closes #2358
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27d70599
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27d70599
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27d70599
Branch: refs/heads/master
Commit: 27d7059984962b97bcaf576fed496653932ea743
Parents: 92d9b92
Author: dhatchayani <dh...@gmail.com>
Authored: Fri Jun 1 15:13:38 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue Jun 5 11:51:52 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 37 +++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27d70599/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1526047..5a7bce3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2688,27 +2688,30 @@ public final class CarbonUtil {
throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
- List<String> listOfFilesRead = new ArrayList<>();
HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
- if (fileStore.getLocationMap() != null) {
+ Map<String, SegmentFileStore.FolderDetails> locationMap = fileStore.getLocationMap();
+ if (locationMap != null) {
fileStore.readIndexFiles();
- Map<String, String> indexFiles = fileStore.getIndexFiles();
Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap();
- for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
- // get the size of carbonindex file
- String indexFile = entry.getKey();
- String mergeIndexFile = indexFiles.get(indexFile);
- if (null != mergeIndexFile) {
- String mergeIndexPath = indexFile
- .substring(0, indexFile.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1)
- + mergeIndexFile;
- if (!listOfFilesRead.contains(mergeIndexPath)) {
- carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize();
- listOfFilesRead.add(mergeIndexPath);
- }
- } else {
- carbonIndexSize += FileFactory.getCarbonFile(indexFile).getSize();
+ // get the size of carbonindex file
+ for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) {
+ SegmentFileStore.FolderDetails folderDetails = entry.getValue();
+ Set<String> carbonindexFiles = folderDetails.getFiles();
+ String mergeFileName = folderDetails.getMergeFileName();
+ if (null != mergeFileName) {
+ String mergeIndexPath =
+ fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR
+ + mergeFileName;
+ carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize();
}
+ for (String indexFile : carbonindexFiles) {
+ String indexPath =
+ fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR
+ + indexFile;
+ carbonIndexSize += FileFactory.getCarbonFile(indexPath).getSize();
+ }
+ }
+ for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
// get the size of carbondata files
for (String blockFile : entry.getValue()) {
carbonDataSize += FileFactory.getCarbonFile(blockFile).getSize();