You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/09 11:17:53 UTC
carbondata git commit: [CARBONDATA-2137] Delete query performance
improved
Repository: carbondata
Updated Branches:
refs/heads/master 1be27b085 -> 7c05f5f18
[CARBONDATA-2137] Delete query performance improved
Following is the configuration used :
SPARK_EXECUTOR_MEMORY : 200G
SPARK_DRIVER_MEMORY : 20G
SPARK_EXECUTOR_CORES : 32
SPARK_EXECUTOR_INSTANCEs : 3
Earlier it was taking 20 minute now it is taking approx 5 minute
This closes #1937
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7c05f5f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7c05f5f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7c05f5f1
Branch: refs/heads/master
Commit: 7c05f5f18a815fdf23de4a703888f9962071fad7
Parents: 1be27b0
Author: rahulforallp <ra...@knoldus.in>
Authored: Tue Feb 6 18:41:35 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Feb 9 16:47:37 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/locks/HdfsFileLock.java | 2 +-
.../carbondata/core/locks/LocalFileLock.java | 2 +-
.../core/mutate/DeleteDeltaBlockDetails.java | 27 +++++++++++++++-----
3 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index 1a46770..cc98b03 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -85,7 +85,7 @@ public class HdfsFileLock extends AbstractCarbonLock {
return true;
} catch (IOException e) {
- LOGGER.error(e, e.getMessage());
+ LOGGER.info(e.getMessage());
return false;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index c3dfb57..4fee4c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -118,7 +118,7 @@ public class LocalFileLock extends AbstractCarbonLock {
return false;
}
} catch (IOException e) {
- LOGGER.error(e, e.getMessage());
+ LOGGER.info(e.getMessage());
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c05f5f1/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index 4275cca..7affe12 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -19,7 +19,10 @@ package org.apache.carbondata.core.mutate;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -31,7 +34,7 @@ public class DeleteDeltaBlockDetails implements Serializable {
private static final long serialVersionUID = 1206104914918495724L;
- private List<DeleteDeltaBlockletDetails> blockletDetails;
+ private Map<String, DeleteDeltaBlockletDetails> blockletDetailsMap;
private String blockName;
/**
@@ -42,7 +45,7 @@ public class DeleteDeltaBlockDetails implements Serializable {
public DeleteDeltaBlockDetails(String blockName) {
this.blockName = blockName;
- blockletDetails = new ArrayList<DeleteDeltaBlockletDetails>();
+ blockletDetailsMap = new TreeMap<>();
}
public String getBlockName() {
@@ -68,15 +71,25 @@ public class DeleteDeltaBlockDetails implements Serializable {
}
public List<DeleteDeltaBlockletDetails> getBlockletDetails() {
- return blockletDetails;
+
+ List<DeleteDeltaBlockletDetails> deleteDeltaBlockletDetailsList = new ArrayList<>();
+ Iterator<Map.Entry<String, DeleteDeltaBlockletDetails>> iterator =
+ blockletDetailsMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ deleteDeltaBlockletDetailsList.add(iterator.next().getValue());
+ }
+ return deleteDeltaBlockletDetailsList;
}
public boolean addBlockletDetails(DeleteDeltaBlockletDetails blocklet) {
- int index = blockletDetails.indexOf(blocklet);
- if (blockletDetails.isEmpty() || index == -1) {
- return blockletDetails.add(blocklet);
+ DeleteDeltaBlockletDetails deleteDeltaBlockletDetails =
+ blockletDetailsMap.get(blocklet.getBlockletKey());
+ if (null == deleteDeltaBlockletDetails) {
+ blockletDetailsMap.put(blocklet.getBlockletKey(), blocklet);
+ return true;
} else {
- return blockletDetails.get(index).addDeletedRows(blocklet.getDeletedRows());
+ deleteDeltaBlockletDetails.addDeletedRows(blocklet.getDeletedRows());
+ return true;
}
}