You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/12/10 13:49:19 UTC
carbondata git commit: [CARBONDATA-3143] Fixed local dictionary in
presto
Repository: carbondata
Updated Branches:
refs/heads/master d9f1a8115 -> 4c9f08217
[CARBONDATA-3143] Fixed local dictionary in presto
Problem:
Currently, local dictionary columns are not working for presto as it is not handled in the integration layer.
Solution:
Add local dictionary support to presto integration layer.
This closes #2972
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4c9f0821
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4c9f0821
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4c9f0821
Branch: refs/heads/master
Commit: 4c9f08217c7b9fa7ad33e148dbf33280e0f2b33f
Parents: d9f1a81
Author: ravipesala <ra...@gmail.com>
Authored: Mon Dec 3 18:27:33 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Mon Dec 10 19:18:32 2018 +0530
----------------------------------------------------------------------
.../presto/CarbonColumnVectorWrapper.java | 2 +-
.../presto/readers/SliceStreamReader.java | 35 +++
.../PrestoAllDataTypeLocalDictTest.scala | 291 +++++++++++++++++++
.../integrationtest/PrestoAllDataTypeTest.scala | 2 +-
.../carbondata/presto/server/PrestoServer.scala | 4 +-
.../presto/util/CarbonDataStoreCreator.scala | 18 +-
6 files changed, 342 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
index a80751f..f001488 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
@@ -244,7 +244,7 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector {
}
@Override public CarbonColumnVector getDictionaryVector() {
- return this.columnVector;
+ return this.columnVector.getDictionaryVector();
}
@Override public void putFloats(int rowId, int count, float[] src, int srcIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
index ab270fc..04e5bb3 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
@@ -17,14 +17,19 @@
package org.apache.carbondata.presto.readers;
+import java.util.Optional;
+
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.DictionaryBlock;
+import com.facebook.presto.spi.block.VariableWidthBlock;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.VarcharType;
+import io.airlift.slice.Slices;
import static io.airlift.slice.Slices.wrappedBuffer;
@@ -63,6 +68,36 @@ public class SliceStreamReader extends CarbonColumnVectorImpl implements PrestoV
}
}
+ @Override public void setDictionary(CarbonDictionary dictionary) {
+ super.setDictionary(dictionary);
+ if (dictionary == null) {
+ dictionaryBlock = null;
+ return;
+ }
+ boolean[] nulls = new boolean[dictionary.getDictionarySize()];
+ nulls[0] = true;
+ nulls[1] = true;
+ int[] dictOffsets = new int[dictionary.getDictionarySize() + 1];
+ int size = 0;
+ for (int i = 0; i < dictionary.getDictionarySize(); i++) {
+ if (dictionary.getDictionaryValue(i) != null) {
+ dictOffsets[i] = size;
+ size += dictionary.getDictionaryValue(i).length;
+ }
+ }
+ byte[] singleArrayDictValues = new byte[size];
+ for (int i = 0; i < dictionary.getDictionarySize(); i++) {
+ if (dictionary.getDictionaryValue(i) != null) {
+ System.arraycopy(dictionary.getDictionaryValue(i), 0, singleArrayDictValues, dictOffsets[i],
+ dictionary.getDictionaryValue(i).length);
+ }
+ }
+ dictOffsets[dictOffsets.length - 1] = size;
+ dictionaryBlock = new VariableWidthBlock(dictionary.getDictionarySize(),
+ Slices.wrappedBuffer(singleArrayDictValues), dictOffsets, Optional.of(nulls));
+ values = (int[]) ((CarbonColumnVectorImpl) getDictionaryVector()).getDataArray();
+ }
+
@Override public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
new file mode 100644
index 0000000..356c0e8
--- /dev/null
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.integrationtest
+
+import java.io.File
+import java.sql.Timestamp
+
+import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.presto.server.PrestoServer
+
+
+class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll {
+
+ private val logger = LogServiceFactory
+ .getLogService(classOf[PrestoAllDataTypeLocalDictTest].getCanonicalName)
+
+ private val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ private val storePath = s"$rootPath/integration/presto/target/store"
+ private val systemPath = s"$rootPath/integration/presto/target/system"
+ private val PrestoServer = new PrestoServer
+
+ // Table schema:
+ // +-------------+----------------+-------------+------------+
+ // | Column name | Data type | Column type | Dictionary |
+ // +-------------+----------------+--------------+-----------+
+ // | id | string | dimension | yes |
+ // +-------------+----------------+-------------+------------+
+ // | date | date | dimension | yes |
+ // +-------------+----------------+-------------+------------+
+ // | country | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | name | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | phonetype | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | serialname | string | dimension | true |
+ // +-------------+----------------+-------------+-------------
+ // | bonus |short decimal | measure | false |
+ // +-------------+----------------+-------------+-------------
+ // | monthlyBonus| longdecimal | measure | false |
+ // +-------------+----------------+-------------+-------------
+ // | dob | timestamp | dimension | true |
+ // +-------------+----------------+-------------+------------+
+ // | shortField | shortfield | measure | true |
+ // +-------------+----------------+-------------+-------------
+ // |isCurrentEmp | boolean | measure | true |
+ // +-------------+----------------+-------------+------------+
+
+ override def beforeAll: Unit = {
+ import org.apache.carbondata.presto.util.CarbonDataStoreCreator
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
+ systemPath)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
+ "Presto")
+ CarbonDataStoreCreator
+ .createCarbonStore(storePath,
+ s"$rootPath/integration/presto/src/test/resources/alldatatype.csv", true)
+ logger.info(s"\nCarbon store is created at location: $storePath")
+ cleanUp
+ PrestoServer.startServer(storePath)
+ }
+
+ override def afterAll(): Unit = {
+ PrestoServer.stopServer()
+ }
+
+ test("select string type with order by clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
+ Map("NAME" -> "anubhav"),
+ Map("NAME" -> "bhavya"),
+ Map("NAME" -> "geetika"),
+ Map("NAME" -> "jatin"),
+ Map("NAME" -> "jitesh"),
+ Map("NAME" -> "liang"),
+ Map("NAME" -> "prince"),
+ Map("NAME" -> "ravindra"),
+ Map("NAME" -> "sahil"),
+ Map("NAME" -> null)
+ )
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test and filter clause with greater than expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 4,
+ "NAME" -> "prince",
+ "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4),
+ "DATE" -> "2015-07-26",
+ "SALARY" -> 15003.0,
+ "SERIALNAME" -> "ASD66902",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2435"),
+ Map("ID" -> 5,
+ "NAME" -> "bhavya",
+ "BONUS" -> java.math.BigDecimal.valueOf(5000.999).setScale(4),
+ "DATE" -> "2015-07-27",
+ "SALARY" -> 15004.0,
+ "SERIALNAME" -> "ASD90633",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2441"))
+ assert(actualResult.toString() equals expectedResult.toString())
+
+
+ }
+
+ test("test and filter clause with greater than equal to expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> "5000000.0",
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"),
+ Map("ID" -> 2,
+ "NAME" -> "jatin",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.5555).setScale(4)
+ ,
+ "DATE" -> "2015-07-24",
+ "SALARY" -> java.math.BigDecimal.valueOf(150010.9990).setScale(3),
+ "SERIALNAME" -> "ASD42892",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone756"),
+ Map("ID" -> 4,
+ "NAME" -> "prince",
+ "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4),
+ "DATE" -> "2015-07-26",
+ "SALARY" -> java.math.BigDecimal.valueOf(15003.0).setScale(1),
+ "SERIALNAME" -> "ASD66902",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2435"),
+ Map("ID" -> 5,
+ "NAME" -> "bhavya",
+ "BONUS" -> java.math.BigDecimal.valueOf(5000.9990).setScale(4),
+ "DATE" -> "2015-07-27",
+ "SALARY" -> java.math.BigDecimal.valueOf(15004.0).setScale(1),
+ "SERIALNAME" -> "ASD90633",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2441"))
+ assert(actualResult.toString() equals expectedResult.toString())
+ }
+ test("test and filter clause with less than equal to expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID LIMIT 2")
+
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> "5000000.0",
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"),
+ Map("ID" -> 3,
+ "NAME" -> "liang",
+ "BONUS" -> java.math.BigDecimal.valueOf(600.7770).setScale(4),
+ "DATE" -> "2015-07-25",
+ "SALARY" -> java.math.BigDecimal.valueOf(15002.11).setScale(2),
+ "SERIALNAME" -> "ASD37014",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone1904"))
+ assert(actualResult.toString() equals expectedResult.toString())
+ }
+ test("test equal to expression on decimal value") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
+
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1))
+
+ assert(actualResult equals expectedResult)
+ }
+ test("test less than expression with and operator") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> 5000000.0,
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"))
+ assert(actualResult.toString().equals(expectedResult.toString()))
+ }
+ test("test the result for in clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("NAME" -> "geetika"),
+ Map("NAME" -> "ravindra"),
+ Map("NAME" -> "jitesh"))
+
+ assert(actualResult.equals(expectedResult))
+ }
+ test("test the result for not in clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
+ Map("NAME" -> "jatin"),
+ Map("NAME" -> "liang"),
+ Map("NAME" -> "prince"),
+ Map("NAME" -> "bhavya"),
+ Map("NAME" -> "akash"),
+ Map("NAME" -> "sahil"))
+
+ assert(actualResult.equals(expectedResult))
+ }
+ test("test for null operator on date data type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
+ assert(actualResult.equals(expectedResult))
+
+ }
+ test("test for not null operator on date data type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
+ assert(actualResult.equals(expectedResult))
+
+ }
+ test("test for not null operator on timestamp type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
+ Map("NAME" -> "jitesh"))
+ assert(actualResult.equals(expectedResult))
+
+ }
+
+ test("test timestamp datatype using cast operator") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ private def cleanUp(): Unit = {
+ FileFactory.deleteFile(s"$storePath/Fact", FileType.LOCAL)
+ FileFactory
+ .createDirectoryAndSetPermission(s"$storePath/_system",
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ FileFactory
+ .createDirectoryAndSetPermission(s"$storePath/.DS_Store",
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ FileFactory.createNewFile(s"$storePath/testdb/.DS_STORE",FileType.LOCAL)
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index d6e322b..97539c0 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
-
+ private val PrestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index 2f3b8f4..1b1accf 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -35,7 +35,7 @@ import org.slf4j.{Logger, LoggerFactory}
import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.presto.CarbondataPlugin
-object PrestoServer {
+class PrestoServer {
val CARBONDATA_CATALOG = "carbondata"
val CARBONDATA_CONNECTOR = "carbondata"
@@ -45,7 +45,7 @@ object PrestoServer {
val prestoProperties: util.Map[String, String] = Map(("http-server.http.port", "8086")).asJava
createSession
- val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
+ lazy val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 4b973a1..f687855 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -67,7 +67,8 @@ object CarbonDataStoreCreator {
/**
* Create store without any restructure
*/
- def createCarbonStore(storePath: String, dataFilePath: String): Unit = {
+ def createCarbonStore(storePath: String, dataFilePath: String,
+ useLocalDict: Boolean = false): Unit = {
try {
logger.info("Creating The Carbon Store")
val dbName: String = "testdb"
@@ -80,7 +81,7 @@ object CarbonDataStoreCreator {
// val factFilePath: String = new File(dataFilePath).getCanonicalPath
val storeDir: File = new File(absoluteTableIdentifier.getTablePath)
CarbonUtil.deleteFoldersAndFiles(storeDir)
- val table: CarbonTable = createTable(absoluteTableIdentifier)
+ val table: CarbonTable = createTable(absoluteTableIdentifier, useLocalDict)
writeDictionary(dataFilePath, table, absoluteTableIdentifier)
val schema: CarbonDataLoadSchema = new CarbonDataLoadSchema(table)
val loadModel: CarbonLoadModel = new CarbonLoadModel()
@@ -141,7 +142,8 @@ object CarbonDataStoreCreator {
}
}
- private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier): CarbonTable = {
+ private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier,
+ useLocalDict: Boolean): CarbonTable = {
val tableInfo: TableInfo = new TableInfo()
tableInfo.setTablePath(absoluteTableIdentifier.getTablePath)
tableInfo.setDatabaseName(
@@ -151,7 +153,9 @@ object CarbonDataStoreCreator {
absoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
val columnSchemas = new ArrayList[ColumnSchema]()
val dictionaryEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
- dictionaryEncoding.add(Encoding.DICTIONARY)
+ if (!useLocalDict) {
+ dictionaryEncoding.add(Encoding.DICTIONARY)
+ }
val invertedIndexEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
invertedIndexEncoding.add(Encoding.INVERTED_INDEX)
@@ -225,9 +229,9 @@ object CarbonDataStoreCreator {
val salary: ColumnSchema = new ColumnSchema()
salary.setColumnName("salary")
salary.setDataType(DataTypes.DOUBLE)
- salary.setEncodingList(dictionaryEncoding)
+ salary.setEncodingList(new util.ArrayList[Encoding]())
salary.setColumnUniqueId(UUID.randomUUID().toString)
- salary.setDimensionColumn(true)
+ salary.setDimensionColumn(false)
salary.setSchemaOrdinal(6)
salary.setColumnReferenceId(salary.getColumnUniqueId)
columnSchemas.add(salary)
@@ -291,6 +295,8 @@ object CarbonDataStoreCreator {
new util.ArrayList[SchemaEvolutionEntry]())
tableSchema.setSchemaEvolution(schemaEvol)
tableSchema.setTableId(UUID.randomUUID().toString)
+ tableSchema.getTableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
+ String.valueOf(useLocalDict))
tableInfo.setTableUniqueName(
absoluteTableIdentifier.getCarbonTableIdentifier.getTableUniqueName
)