You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/12/05 13:21:33 UTC
[1/2] incubator-carbondata git commit: Added unit test for
DictionaryBasedResultCollector
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 87f745c60 -> 0a2841a2f
Added unit test for DictionaryBasedResultCollector
added unit test for RawBasedResultCollector
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/247f2027
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/247f2027
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/247f2027
Branch: refs/heads/master
Commit: 247f2027e6091fc825b31c97461de7b8dea827c4
Parents: 87f745c
Author: kunal642 <ku...@knoldus.in>
Authored: Wed Nov 23 16:32:43 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Dec 5 18:50:20 2016 +0530
----------------------------------------------------------------------
.../DictionaryBasedResultCollectorTest.java | 190 +++++++++++++++++++
.../impl/RawBasedResultCollectorTest.java | 158 +++++++++++++++
2 files changed, 348 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/247f2027/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
new file mode 100644
index 0000000..8900cef
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.scan.collector.impl;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.scan.complextypes.ArrayQueryType;
+import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.scan.filter.GenericQueryType;
+import org.apache.carbondata.scan.model.QueryDimension;
+import org.apache.carbondata.scan.model.QueryMeasure;
+import org.apache.carbondata.scan.result.AbstractScannedResult;
+import org.apache.carbondata.scan.result.impl.NonFilterQueryScannedResult;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class DictionaryBasedResultCollectorTest {
+
+ private static DictionaryBasedResultCollector dictionaryBasedResultCollector;
+ private static BlockExecutionInfo blockExecutionInfo;
+
+ @BeforeClass public static void setUp() {
+ blockExecutionInfo = new BlockExecutionInfo();
+ KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
+ blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
+ AggregatorInfo aggregatorInfo = new AggregatorInfo();
+ aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
+ aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
+ aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
+ aggregatorInfo.setMeasureDataTypes(
+ new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
+ blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+ QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+ queryDimension1.setQueryOrder(1);
+ ColumnSchema columnSchema = new ColumnSchema();
+ queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
+ QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+ queryDimension2.setQueryOrder(2);
+ queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
+ QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+ queryDimension3.setQueryOrder(3);
+ queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
+ QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+ queryDimension4.setQueryOrder(4);
+ queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
+ blockExecutionInfo.setQueryDimensions(
+ new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+ queryDimension4 });
+ QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+ queryMeasure1.setQueryOrder(1);
+ QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+ queryMeasure1.setQueryOrder(2);
+ QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+ queryMeasure1.setQueryOrder(3);
+ QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+ queryMeasure1.setQueryOrder(4);
+ blockExecutionInfo.setQueryMeasures(
+ new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+ Map<Integer, GenericQueryType> complexDimensionInfoMap = new HashMap<>();
+ complexDimensionInfoMap.put(1, new ArrayQueryType("name1", "parent1", 1));
+ complexDimensionInfoMap.put(2, new ArrayQueryType("name2", "parent2", 2));
+ complexDimensionInfoMap.put(3, new ArrayQueryType("name3", "parent3", 3));
+ complexDimensionInfoMap.put(4, new ArrayQueryType("name4", "parent4", 4));
+ blockExecutionInfo.setComplexDimensionInfoMap(complexDimensionInfoMap);
+ dictionaryBasedResultCollector = new DictionaryBasedResultCollector(blockExecutionInfo);
+ }
+
+ @Test public void testToCollectData() {
+ new MockUp<CarbonUtil>() {
+ @SuppressWarnings("unused") @Mock boolean[] getDictionaryEncodingArray(
+ QueryDimension[] queryDimensions) {
+ return new boolean[] { true, false, true, true };
+ }
+
+ @SuppressWarnings("unused") @Mock boolean[] getDirectDictionaryEncodingArray(
+ QueryDimension[] queryDimensions) {
+ return new boolean[] { true, true, false, false };
+ }
+
+ @SuppressWarnings("unused") @Mock boolean[] getComplexDataTypeArray(
+ QueryDimension[] queryDimensions) {
+ return new boolean[] { false, false, true, false };
+ }
+ };
+ new MockUp<DataTypeUtil>() {
+ @SuppressWarnings("unused") @Mock Object getDataBasedOnDataType(String data,
+ DataType actualDataType) {
+ return 1;
+ }
+ };
+
+ new MockUp<NonFilterQueryScannedResult>() {
+ @SuppressWarnings("unused") @Mock int[] getDictionaryKeyIntegerArray() {
+ this.getMockInstance().incrementCounter();
+ System.out.println("Mocked");
+ return new int[] { 1, 2 };
+ }
+
+ @SuppressWarnings("unused") @Mock String[] getNoDictionaryKeyStringArray() {
+ return new String[] { "1", "2" };
+ }
+
+ @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
+ return new byte[][] { { 1, 2 }, { 1, 2 } };
+ }
+
+ @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+ MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+ PresenceMeta presenceMeta = new PresenceMeta();
+ BitSet bitSet = new BitSet();
+ bitSet.set(1);
+ presenceMeta.setBitSet(bitSet);
+ measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
+ CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
+ carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
+ measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
+ return measureColumnDataChunk;
+ }
+ };
+
+ new MockUp<DirectDictionaryKeyGeneratorFactory>() {
+ @SuppressWarnings("unused") @Mock DirectDictionaryGenerator getDirectDictionaryGenerator(
+ DataType dataType) {
+ if (dataType == DataType.TIMESTAMP) return new TimeStampDirectDictionaryGenerator(
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ else return null;
+ }
+ };
+ new MockUp<TimeStampDirectDictionaryGenerator>() {
+ @SuppressWarnings("unused") @Mock Object getValueFromSurrogate(int key) {
+ return 100L;
+ }
+ };
+
+ new MockUp<ArrayQueryType>() {
+ @SuppressWarnings("unused") @Mock Object getDataBasedOnDataTypeFromSurrogates(
+ ByteBuffer surrogateData) {
+ return ByteBuffer.wrap("1".getBytes());
+ }
+ };
+
+ AbstractScannedResult abstractScannedResult =
+ new NonFilterQueryScannedResult(blockExecutionInfo);
+ abstractScannedResult.setNumberOfRows(2);
+ List<Object[]> result = dictionaryBasedResultCollector.collectData(abstractScannedResult, 2);
+ int expectedResult = 2;
+ assertThat(result.size(), is(equalTo(expectedResult)));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/247f2027/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
new file mode 100644
index 0000000..78b5916
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.scan.collector.impl;
+
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.scan.executor.util.QueryUtil;
+import org.apache.carbondata.scan.model.QueryDimension;
+import org.apache.carbondata.scan.model.QueryMeasure;
+import org.apache.carbondata.scan.result.AbstractScannedResult;
+import org.apache.carbondata.scan.result.impl.NonFilterQueryScannedResult;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class RawBasedResultCollectorTest {
+
+ private static RawBasedResultCollector rawBasedResultCollector;
+ private static BlockExecutionInfo blockExecutionInfo;
+ private static KeyGenerator keyGenerator;
+
+ @BeforeClass public static void setUp() {
+ keyGenerator = new MockUp<KeyGenerator>() {
+ @SuppressWarnings("unused") @Mock long[] getKeyArray(byte[] key, int[] maskedByteRanges) {
+ return new long[] { 1, 2 };
+ }
+
+ @SuppressWarnings("unused") @Mock byte[] generateKey(long[] keys) throws KeyGenException {
+ return new byte[] { 1, 2 };
+ }
+
+ }.getMockInstance();
+
+ blockExecutionInfo = new BlockExecutionInfo();
+ KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
+ keyStructureInfo.setKeyGenerator(keyGenerator);
+ keyStructureInfo.setMaxKey(new byte[] { 1, 2 });
+ keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
+ keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
+ blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
+ AggregatorInfo aggregatorInfo = new AggregatorInfo();
+ aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
+ aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
+ aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
+ aggregatorInfo.setMeasureDataTypes(
+ new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
+ QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+ queryMeasure1.setQueryOrder(1);
+ QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+ queryMeasure1.setQueryOrder(2);
+ QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+ queryMeasure1.setQueryOrder(3);
+ QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+ queryMeasure1.setQueryOrder(4);
+ QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+ queryDimension1.setQueryOrder(1);
+ ColumnSchema columnSchema = new ColumnSchema();
+ queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
+ QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+ queryDimension2.setQueryOrder(2);
+ queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
+ QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+ queryDimension3.setQueryOrder(3);
+ queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
+ QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+ queryDimension4.setQueryOrder(4);
+ queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
+ blockExecutionInfo.setQueryDimensions(
+ new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+ queryDimension4 });
+ blockExecutionInfo.setQueryMeasures(
+ new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+ blockExecutionInfo.setFixedKeyUpdateRequired(true);
+ blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+ blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
+ blockExecutionInfo.setBlockKeyGenerator(keyGenerator);
+ rawBasedResultCollector = new RawBasedResultCollector(blockExecutionInfo);
+ }
+
+ @Test public void testToCollectData() {
+
+ new MockUp<NonFilterQueryScannedResult>() {
+ @SuppressWarnings("unused") @Mock byte[] getDictionaryKeyArray() {
+ this.getMockInstance().incrementCounter();
+ return new byte[] { 1, 2 };
+ }
+
+ @SuppressWarnings("unused") @Mock byte[][] getNoDictionaryKeyArray() {
+ return new byte[][] { { 1, 2 } };
+ }
+
+ @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
+ return new byte[][] { { 1, 2 }, { 1, 2 } };
+ }
+
+ @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+ MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+ PresenceMeta presenceMeta = new PresenceMeta();
+ BitSet bitSet = new BitSet();
+ bitSet.set(1);
+ presenceMeta.setBitSet(bitSet);
+ measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
+ CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
+ carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
+ measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
+ return measureColumnDataChunk;
+ }
+ };
+
+ new MockUp<QueryUtil>() {
+ @SuppressWarnings("unused") @Mock byte[] getMaskedKey(byte[] data, byte[] maxKey,
+ int[] maskByteRanges, int byteCount) {
+ return new byte[] { 1, 2 };
+ }
+ };
+
+ AbstractScannedResult abstractScannedResult =
+ new NonFilterQueryScannedResult(blockExecutionInfo);
+ abstractScannedResult.setNumberOfRows(2);
+ List<Object[]> result = rawBasedResultCollector.collectData(abstractScannedResult, 2);
+ int expectedResult = 2;
+ assertThat(result.size(), is(equalTo(expectedResult)));
+ }
+}
[2/2] incubator-carbondata git commit: [CARBONDATA-446] Added unit
test for scan.collector.impl package This closes #349
Posted by ra...@apache.org.
[CARBONDATA-446] Added unit test for scan.collector.impl package This closes #349
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0a2841a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0a2841a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0a2841a2
Branch: refs/heads/master
Commit: 0a2841a2f04c8d1391ca3798d2668cb674aa267b
Parents: 87f745c 247f202
Author: ravipesala <ra...@gmail.com>
Authored: Mon Dec 5 18:51:17 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Dec 5 18:51:17 2016 +0530
----------------------------------------------------------------------
.../DictionaryBasedResultCollectorTest.java | 190 +++++++++++++++++++
.../impl/RawBasedResultCollectorTest.java | 158 +++++++++++++++
2 files changed, 348 insertions(+)
----------------------------------------------------------------------