You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/07/07 21:31:08 UTC
[1/2] incubator-carbondata git commit: Segment folders are deleted
for multiple loads
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 61a4355f6 -> 8dfa67238
Segment folders are deleted for multiple loads
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a156737e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a156737e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a156737e
Branch: refs/heads/master
Commit: a156737e1dc992f42a796d15e1891a2e6ba5388e
Parents: 61a4355
Author: ravipesala <ra...@gmail.com>
Authored: Fri Jul 8 00:04:14 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jul 8 00:04:14 2016 +0530
----------------------------------------------------------------------
.../carbondata/spark/load/CarbonLoaderUtil.java | 23 ++++++++++++--------
1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a156737e/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index af880f5..52f8fb7 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -307,25 +307,30 @@ public final class CarbonLoaderUtil {
String metaDataLocation = carbonTable.getMetaDataFilepath();
SegmentStatusManager segmentStatusManager =
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
- LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
+ final LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
- final List<String> loadFolders = new ArrayList<String>();
- for (LoadMetadataDetails loadMetadata : details) {
- loadFolders.add(carbonTablePath
- .getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(), loadMetadata.getLoadName())
- .replace("\\", "/"));
- }
//delete folder which metadata no exist in tablestatus
for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- String partitionPath = carbonTablePath.getPartitionDir(i + "");
+ final String partitionCount = i + "";
+ String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
FileType fileType = FileFactory.getFileType(partitionPath);
if (FileFactory.isFileExist(partitionPath, fileType)) {
CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile path) {
- return !loadFolders.contains(path.getAbsolutePath().replace("\\", "/"));
+ String segmentId =
+ CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
+ boolean found = false;
+ for (int j = 0; j < details.length; j++) {
+ if (details[j].getLoadName().equals(segmentId)
+ && details[j].getPartitionCount().equals(partitionCount)) {
+ found = true;
+ break;
+ }
+ }
+ return !found;
}
});
for (int k = 0; k < listFiles.length; k++) {
[2/2] incubator-carbondata git commit: [CARBONDATA-44]Segment folders
are deleted for multiple loads This closes #29
Posted by ch...@apache.org.
[CARBONDATA-44]Segment folders are deleted for multiple loads This closes #29
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8dfa6723
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8dfa6723
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8dfa6723
Branch: refs/heads/master
Commit: 8dfa6723813f925efe27fdc37fefd6e4cc539088
Parents: 61a4355 a156737
Author: chenliang613 <ch...@apache.org>
Authored: Fri Jul 8 03:00:52 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Fri Jul 8 03:00:52 2016 +0530
----------------------------------------------------------------------
.../carbondata/spark/load/CarbonLoaderUtil.java | 23 ++++++++++++--------
1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------