You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2017/05/31 07:24:16 UTC
[1/2] carbondata git commit: use binarySearch to replace for clause
to improve performance
Repository: carbondata
Updated Branches:
refs/heads/master 84fb72439 -> 7b7579b87
use binarySearch to replace for clause to improve performance
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a5e3643c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a5e3643c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a5e3643c
Branch: refs/heads/master
Commit: a5e3643c4ac8ed8a02edfcba44c0cb56d1da991d
Parents: 84fb724
Author: mayun <si...@163.com>
Authored: Wed May 24 14:04:43 2017 +0800
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed May 31 12:51:07 2017 +0530
----------------------------------------------------------------------
.../executer/ExcludeFilterExecuterImpl.java | 13 +++-
.../executer/ExcludeFilterExecuterImplTest.java | 63 ++++++++++++++++++++
.../executer/IncludeFilterExecuterImplTest.java | 16 ++---
3 files changed, 82 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a5e3643c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 8e7a3c2..7449781 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -141,14 +141,23 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
+ // use binary search to replace for clause
private BitSet setFilterdIndexToBitSet(FixedLengthDimensionDataChunk dimColumnDataChunk,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int k = 0; k < filterValues.length; k++) {
+ if (filterValues.length > 1) {
for (int j = 0; j < numerOfRows; j++) {
- if (dimColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
+ dimColumnDataChunk.getChunkData(j));
+ if (index >= 0) {
+ bitSet.flip(j);
+ }
+ }
+ } else if (filterValues.length == 1) {
+ for (int j = 0; j < numerOfRows; j++) {
+ if (dimColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
bitSet.flip(j);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a5e3643c/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
new file mode 100644
index 0000000..e3ae42c
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest {
+
+ @Override public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimColumnDataChunk,
+ int numerOfRows, byte[][] filterValues) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ // byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ if (filterValues.length > 1) {
+ for (int j = 0; j < numerOfRows; j++) {
+ int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
+ dimColumnDataChunk.getChunkData(j));
+ if (index >= 0) {
+ bitSet.flip(j);
+ }
+ }
+ } else if (filterValues.length == 1) {
+ for (int j = 0; j < numerOfRows; j++) {
+ if (dimColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ bitSet.flip(j);
+ }
+ }
+ }
+ return bitSet;
+ }
+
+ @Override public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimColumnDataChunk,
+ int numerOfRows, byte[][] filterValues) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ // byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int k = 0; k < filterValues.length; k++) {
+ for (int j = 0; j < numerOfRows; j++) {
+ if (dimColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ bitSet.flip(j);
+ }
+ }
+ }
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a5e3643c/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
index 87b9c2d..404f77f 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
@@ -36,7 +36,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
- private BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimensionColumnDataChunk,
+ public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimensionColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
@@ -60,7 +60,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
@@ -99,8 +99,8 @@ public class IncludeFilterExecuterImplTest extends TestCase {
@Test
public void testPerformance() {
- // dimension's data number in a blocklet, usually default is 120000
- int dataChunkSize = 120000;
+ // dimension's data number in a blocklet, usually default is 32000
+ int dataChunkSize = 32000;
// repeat query times in the test
int queryTimes = 5;
// repeated times for a dictionary value
@@ -122,8 +122,8 @@ public class IncludeFilterExecuterImplTest extends TestCase {
@Test
public void testBoundary() {
- // dimension's data number in a blocklet, usually default is 120000
- int dataChunkSize = 120000;
+ // dimension's data number in a blocklet, usually default is 32000
+ int dataChunkSize = 32000;
// repeat query times in the test
int queryTimes = 5;
// repeated times for a dictionary value
@@ -268,8 +268,8 @@ public class IncludeFilterExecuterImplTest extends TestCase {
long start;
long end;
- // dimension's data number in a blocklet, usually default is 120000
- int dataChunkSize = 120000;
+ // dimension's data number in a blocklet, usually default is 32000
+ int dataChunkSize = 32000;
// repeat query times in the test
int queryTimes = 10000;
// repeated times for a dictionary value
[2/2] carbondata git commit: [CARBONDATA-1081] Use binarySearch to
replace for clause to improve performance This closes #940
Posted by ku...@apache.org.
[CARBONDATA-1081] Use binarySearch to replace for clause to improve performance This closes #940
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7b7579b8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7b7579b8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7b7579b8
Branch: refs/heads/master
Commit: 7b7579b87fa06709f8edf4acf0d7dc5e77875624
Parents: 84fb724 a5e3643
Author: kumarvishal <ku...@gmail.com>
Authored: Wed May 31 12:53:50 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed May 31 12:53:50 2017 +0530
----------------------------------------------------------------------
.../executer/ExcludeFilterExecuterImpl.java | 13 +++-
.../executer/ExcludeFilterExecuterImplTest.java | 63 ++++++++++++++++++++
.../executer/IncludeFilterExecuterImplTest.java | 16 ++---
3 files changed, 82 insertions(+), 10 deletions(-)
----------------------------------------------------------------------