You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:40:56 UTC
[31/50] [abbrv] carbondata git commit: [HOTFIX] Changes in selecting
the carbonindex files
[HOTFIX] Changes in selecting the carbonindex files
Currently, in the query flow while getting the index files we are checking for either mergeFileName or the list of files. After this change, we will
be checking for both files and mergeFileName
This closes #2333
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/22d5035c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/22d5035c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/22d5035c
Branch: refs/heads/spark-2.3
Commit: 22d5035c84342e0c0b15a87abbdd4dca5e6d4976
Parents: d777318
Author: dhatchayani <dh...@gmail.com>
Authored: Tue May 22 17:26:37 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue May 29 11:01:08 2018 +0530
----------------------------------------------------------------------
.../core/metadata/SegmentFileStore.java | 18 ++++++++++++------
.../core/writer/CarbonIndexFileMergeWriter.java | 2 ++
2 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22d5035c/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index d72ded3..acfc145 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -151,7 +151,8 @@ public class SegmentFileStore {
CarbonFile segmentFolder = FileFactory.getCarbonFile(segmentPath);
CarbonFile[] indexFiles = segmentFolder.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
- return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT);
+ return (file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || file.getName()
+ .endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT));
}
});
if (indexFiles != null && indexFiles.length > 0) {
@@ -160,7 +161,11 @@ public class SegmentFileStore {
folderDetails.setRelative(true);
folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
for (CarbonFile file : indexFiles) {
- folderDetails.getFiles().add(file.getName());
+ if (file.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ folderDetails.setMergeFileName(file.getName());
+ } else {
+ folderDetails.getFiles().add(file.getName());
+ }
}
String segmentRelativePath = segmentPath.substring(tablePath.length(), segmentPath.length());
segmentFile.addPath(segmentRelativePath, folderDetails);
@@ -508,10 +513,11 @@ public class SegmentFileStore {
if (null != mergeFileName) {
indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + mergeFileName,
entry.getValue().mergeFileName);
- } else {
- for (String indexFile : entry.getValue().getFiles()) {
- indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile,
- entry.getValue().mergeFileName);
+ }
+ Set<String> files = entry.getValue().getFiles();
+ if (null != files && !files.isEmpty()) {
+ for (String indexFile : files) {
+ indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22d5035c/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index ceeb431..cb53c0b 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -21,6 +21,7 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -140,6 +141,7 @@ public class CarbonIndexFileMergeWriter {
}
if (new Path(entry.getKey()).equals(new Path(location))) {
segentry.getValue().setMergeFileName(mergeIndexFile);
+ segentry.getValue().setFiles(new HashSet<String>());
break;
}
}