You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/24 13:26:15 UTC
[19/50] [abbrv] carbondata git commit: [CARBONDATA-1907] Avoid
unnecessary logging to improve query performance for no dictionary non string
columns
[CARBONDATA-1907] Avoid unnecessary logging to improve query performance for no dictionary non string columns
Changes done to return null in case of no dictionary column for non string data types when data is empty.This is done to avoid excessive logging which is impacting the query performance.
This closes #1679
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a86c54b3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a86c54b3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a86c54b3
Branch: refs/heads/fgdatamap
Commit: a86c54b3e83fef0b45ed70c50ee7483f5cc0894c
Parents: 694ee77
Author: manishgupta88 <to...@gmail.com>
Authored: Tue Dec 19 17:27:30 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed Dec 20 14:06:09 2017 +0530
----------------------------------------------------------------------
.../executor/impl/AbstractQueryExecutor.java | 11 ++++-----
.../carbondata/core/util/DataTypeUtil.java | 24 ++++++++++++++++++++
.../carbondata/core/util/DataTypeUtilTest.java | 18 +++++++++++++++
3 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 88a8238..cb193e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
@@ -544,12 +543,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
}
UnsafeMemoryManager.INSTANCE.freeMemoryAll(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId());
if (null != queryProperties.executorService) {
- queryProperties.executorService.shutdown();
- try {
- queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new QueryExecutionException(e);
- }
+ // In case of limit query when number of limit records is already found so executors
+ // must stop all the running execution otherwise it will keep running and will hit
+ // the query performance.
+ queryProperties.executorService.shutdownNow();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8c13a3..0e819c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -353,12 +353,26 @@ public final class DataTypeUtil {
} else if (actualDataType == DataTypes.STRING) {
return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
} else if (actualDataType == DataTypes.SHORT) {
+ // for non string type no dictionary column empty byte array is empty value
+ // so no need to parse
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.INT) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.LONG) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.TIMESTAMP) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length) * 1000L;
} else {
return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
@@ -372,6 +386,16 @@ public final class DataTypeUtil {
}
}
+ /**
+ * Method to check if byte array is empty
+ *
+ * @param dataInBytes
+ * @return
+ */
+ private static boolean isEmptyByteArray(byte[] dataInBytes) {
+ return dataInBytes.length == 0;
+ }
+
/**
* Below method will be used to convert the data passed to its actual data
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index c944d18..bb528b9 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -98,6 +98,24 @@ public class DataTypeUtilTest {
}
+ @Test public void testGetDataBasedOnDataTypeForNoDictionaryColumn() {
+ Object result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.INT);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.SHORT);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.LONG);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.TIMESTAMP);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.STRING);
+ assert (result != null);
+ }
+
}