You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/12/13 14:46:32 UTC
carbondata git commit: [CARBONDATA-3158] support presto-carbon to
read sdk cabron files
Repository: carbondata
Updated Branches:
refs/heads/master 88fa067e0 -> db08fe1f8
[CARBONDATA-3158] support presto-carbon to read sdk cabron files
problem:
Currently, carbon SDK files output (files without metadata folder and its
contents) are read by spark using an external table with carbon session.
But presto carbon integration doesn't support that. It can currently read
only the transactional table output files.
solution:
Hence we can enhance presto to read SDK output files. This will increase
the use cases for presto-carbon integration.
The above scenario can be achieved by inferring schema if metadata folder
not exists and
setting read committed scope to LatestFilesReadCommittedScope, if
non-transctional table output files are present.
This closes #2982
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/db08fe1f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/db08fe1f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/db08fe1f
Branch: refs/heads/master
Commit: db08fe1f86acc7b6db7f1207aed062cb4ee42aaf
Parents: 88fa067
Author: ajantha-bhat <aj...@gmail.com>
Authored: Fri Dec 7 18:37:10 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Dec 13 20:08:17 2018 +0530
----------------------------------------------------------------------
integration/presto/pom.xml | 6 +
.../presto/CarbondataPageSourceProvider.java | 3 +
.../presto/impl/CarbonTableReader.java | 74 +++---
.../PrestoAllDataTypeLocalDictTest.scala | 30 +--
.../integrationtest/PrestoAllDataTypeTest.scala | 102 ++++----
.../PrestoTestNonTransactionalTableFiles.scala | 233 +++++++++++++++++++
.../carbondata/presto/server/PrestoServer.scala | 35 ++-
7 files changed, 378 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 31a4de6..3d7ec44 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -487,6 +487,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-store-sdk</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>1.4.0</version>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
index bef246e..96024e4 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
@@ -139,6 +139,9 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
Configuration conf = new Configuration();
conf.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
+ CarbonTableInputFormat
+ .setTransactionalTable(conf, carbonTable.getTableInfo().isTransactionalTable());
+ CarbonTableInputFormat.setTableInfo(conf, carbonTable.getTableInfo());
conf.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
conf.set("query.id", queryId);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 546bf9b..9677839 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -53,8 +53,10 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.api.CarbonInputFormat;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
import org.apache.carbondata.presto.PrestoFilterUtil;
@@ -284,18 +286,6 @@ public class CarbonTableReader {
if (carbonFileList == null) {
updateSchemaList();
}
- try {
- if (isKeyExists
- && !FileFactory.isFileExist(
- CarbonTablePath.getSchemaFilePath(
- carbonCache.get().get(schemaTableName).carbonTable.getTablePath()), fileType)) {
- removeTableFromCache(schemaTableName);
- throw new TableNotFoundException(schemaTableName);
- }
- } catch (IOException e) {
- throw new RuntimeException();
- }
-
if (isKeyExists) {
CarbonTableCacheModel carbonTableCacheModel = carbonCache.get().get(schemaTableName);
if (carbonTableCacheModel != null
@@ -345,14 +335,15 @@ public class CarbonTableReader {
* @return the CarbonTable instance which contains all the needed metadata for a table.
*/
private CarbonTable parseCarbonMetadata(SchemaTableName table) {
- CarbonTable result = null;
+ CarbonTable result;
try {
CarbonTableCacheModel cache = carbonCache.get().get(table);
if (cache == null) {
cache = new CarbonTableCacheModel();
}
- if (cache.isValid()) return cache.carbonTable;
-
+ if (cache.isValid()) {
+ return cache.carbonTable;
+ }
// If table is not previously cached, then:
// Step 1: get store path of the table and cache it.
@@ -364,23 +355,38 @@ public class CarbonTableReader {
String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/"
+ carbonTableIdentifier.getTableName();
- //Step 2: read the metadata (tableInfo) of the table.
- ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
- // TBase is used to read and write thrift objects.
- // TableInfo is a kind of TBase used to read and write table information.
- // TableInfo is generated by thrift,
- // see schema.thrift under format/src/main/thrift for details.
- public TBase create() {
- return new org.apache.carbondata.format.TableInfo();
+ String metadataPath = CarbonTablePath.getSchemaFilePath(tablePath);
+ boolean isTransactionalTable = false;
+ try {
+ FileFactory.FileType fileType = FileFactory.getFileType(metadataPath);
+ if (FileFactory.getCarbonFile(metadataPath, fileType).isFileExist(metadataPath, fileType)) {
+ // If metadata folder exists, it is a transactional table
+ isTransactionalTable = true;
}
- };
- ThriftReader thriftReader =
- new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
- thriftReader.open();
- org.apache.carbondata.format.TableInfo tableInfo =
- (org.apache.carbondata.format.TableInfo) thriftReader.read();
- thriftReader.close();
-
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ org.apache.carbondata.format.TableInfo tableInfo;
+ if (isTransactionalTable) {
+ //Step 2: read the metadata (tableInfo) of the table.
+ ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
+ // TBase is used to read and write thrift objects.
+ // TableInfo is a kind of TBase used to read and write table information.
+ // TableInfo is generated by thrift,
+ // see schema.thrift under format/src/main/thrift for details.
+ public TBase create() {
+ return new org.apache.carbondata.format.TableInfo();
+ }
+ };
+ ThriftReader thriftReader =
+ new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
+ thriftReader.open();
+ tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
+ thriftReader.close();
+ } else {
+ tableInfo = CarbonUtil
+ .inferSchema(tablePath, table.getTableName(), false, FileFactory.getConfiguration());
+ }
// Step 3: convert format level TableInfo to code level TableInfo
SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
// wrapperTableInfo is the code level information of a table in carbondata core,
@@ -389,6 +395,8 @@ public class CarbonTableReader {
.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(),
tablePath);
+ wrapperTableInfo.setTransactionalTable(isTransactionalTable);
+
// Step 4: Load metadata info into CarbonMetadata
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
@@ -419,6 +427,8 @@ public class CarbonTableReader {
config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
config.set("query.id", queryId);
+ CarbonInputFormat.setTransactionalTable(config, carbonTable.isTransactionalTable());
+ CarbonInputFormat.setTableInfo(config, carbonTable.getTableInfo());
JobConf jobConf = new JobConf(config);
List<PartitionSpec> filteredPartitions = new ArrayList();
@@ -470,7 +480,7 @@ public class CarbonTableReader {
}
} catch (IOException e) {
- throw new RuntimeException("Error creating Splits from CarbonTableInputFormat", e);
+ throw new RuntimeException(e);
}
return multiBlockSplitList;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
index 356c0e8..3f4d839 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
- private val PrestoServer = new PrestoServer
+ private val prestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
@@ -80,15 +80,15 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
s"$rootPath/integration/presto/src/test/resources/alldatatype.csv", true)
logger.info(s"\nCarbon store is created at location: $storePath")
cleanUp
- PrestoServer.startServer(storePath)
+ prestoServer.startServer(storePath)
}
override def afterAll(): Unit = {
- PrestoServer.stopServer()
+ prestoServer.stopServer()
}
test("select string type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
Map("NAME" -> "anubhav"),
@@ -106,7 +106,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test and filter clause with greater than expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -133,7 +133,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test and filter clause with greater than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -174,7 +174,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString() equals expectedResult.toString())
}
test("test and filter clause with less than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -199,7 +199,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString() equals expectedResult.toString())
}
test("test equal to expression on decimal value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
@@ -208,7 +208,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult equals expectedResult)
}
test("test less than expression with and operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -224,7 +224,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(
Map("NAME" -> "geetika"),
@@ -234,7 +234,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.equals(expectedResult))
}
test("test the result for not in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
@@ -248,21 +248,21 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.equals(expectedResult))
}
test("test for null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
Map("NAME" -> "jitesh"))
@@ -271,7 +271,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test timestamp datatype using cast operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
assert(actualResult.equals(expectedResult))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 97539c0..7bab795 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
- private val PrestoServer = new PrestoServer
+ private val prestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
@@ -80,77 +80,77 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
logger.info(s"\nCarbon store is created at location: $storePath")
cleanUp
- PrestoServer.startServer(storePath)
+ prestoServer.startServer(storePath, "testdb")
}
override def afterAll(): Unit = {
- PrestoServer.stopServer()
+ prestoServer.stopServer()
}
test("test the result for count(*) in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(*) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 11))
assert(actualResult.equals(expectedResult))
}
test("test the result for count() clause with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 9))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum()in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 54))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum() wiTh distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 45))
assert(actualResult.equals(expectedResult))
}
test("test the result for avg() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT AVG(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 5))
assert(actualResult.equals(expectedResult))
}
test("test the result for min() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MIN(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 1))
assert(actualResult.equals(expectedResult))
}
test("test the result for max() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MAX(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 9))
assert(actualResult.equals(expectedResult))
}
test("test the result for count()clause with distinct operator on decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
assert(actualResult.equals(expectedResult))
}
test("test the result for count()clause with out distinct operator on decimal column in presto")
{
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum()with out distinct operator for decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475))
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for sum() with distinct operator for decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475))
assert(
@@ -158,14 +158,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
expectedResult.head("RESULT").toString.toDouble)
}
test("test the result for avg() with distinct operator on decimal on presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT AVG(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 2077.4648))
assert(actualResult.toString.equals(expectedResult.toString))
}
test("test the result for min() with distinct operator in decimalType of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MIN(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map(
"RESULT" -> java.math.BigDecimal.valueOf(500.414).setScale(4)))
@@ -173,14 +173,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the result for max() with distinct operator in decimalType of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MAX(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map(
"RESULT" -> java.math.BigDecimal.valueOf(9999.999).setScale(4)))
assert(actualResult.equals(expectedResult))
}
test("select decimal data type with ORDER BY clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DISTINCT BONUS FROM TESTDB.TESTTABLE ORDER BY BONUS limit 3 ")
val expectedResult: List[Map[String, Any]] = List(Map(
"BONUS" -> java.math.BigDecimal.valueOf(500.414).setScale(4)),
@@ -189,7 +189,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("select string type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
Map("NAME" -> "anubhav"),
@@ -206,7 +206,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("select DATE type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DATE FROM TESTDB.TESTTABLE ORDER BY DATE")
val expectedResult: List[Map[String, Any]] = List(Map("DATE" -> "2015-07-18"),
Map("DATE" -> "2015-07-23"),
@@ -228,7 +228,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.reverse.head("DATE") == null)
}
test("select int type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DISTINCT ID FROM TESTDB.TESTTABLE ORDER BY ID")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1),
Map("ID" -> 2),
@@ -247,7 +247,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test and filter clause with greater than expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -274,7 +274,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test and filter clause with greater than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -315,7 +315,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString() equals expectedResult.toString())
}
test("test and filter clause with less than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -340,7 +340,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString() equals expectedResult.toString())
}
test("test equal to expression on decimal value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
@@ -349,7 +349,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult equals expectedResult)
}
test("test less than expression with and operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -365,7 +365,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(
Map("NAME" -> "geetika"),
@@ -375,7 +375,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for not in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
@@ -389,21 +389,21 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test for null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
Map("NAME" -> "jitesh"))
@@ -411,14 +411,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test for null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NULL AND ID=1")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"))
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT DISTINCT SHORTFIELD from testdb.testtable ORDER BY SHORTFIELD ")
val expectedResult: List[Map[String, Any]] = List(Map("SHORTFIELD" -> 1),
@@ -433,7 +433,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype in clause where field is null") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE SHORTFIELD IS NULL ORDER BY SHORTFIELD ")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 7),Map("ID" -> null))
@@ -441,7 +441,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype with greater than operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE SHORTFIELD>11 ")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 6), Map("ID" -> 9))
@@ -450,7 +450,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test longDecimal type of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE bonus = DECIMAL '1234.5555'")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 2))
@@ -459,7 +459,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test shortDecimal type of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE monthlyBonus = 15.13")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 2))
@@ -468,42 +468,42 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test timestamp datatype using cast operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
assert(actualResult.equals(expectedResult))
}
test("test timestamp datatype using cast and in operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID AS RESULT FROM TESTDB.TESTTABLE WHERE DOB in (cast('2016-04-14 " +
"15:00:09' as timestamp),cast('2015-10-07' as timestamp),cast('2015-10-07 01:00:03' as timestamp))")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "2"))
assert(actualResult.toString() equals expectedResult.toString())
}
test("test the boolean data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT isCurrentEmployee AS RESULT FROM TESTDB.TESTTABLE WHERE ID=1")
assert(actualResult.head("RESULT").toString.toBoolean)
}
test("test the boolean data type for null value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is null")
assert(actualResult.head("RESULT").toString.toInt==2)
}
test("test the boolean data type for not null value with filter ") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is NOT null AND ID>8")
assert(actualResult.head("RESULT").toString.toInt==9)
}
test("test the is null operator when null is included in string data type dictionary_include"){
// See CARBONDATA-2155
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SERIALNAME FROM TESTDB.TESTTABLE WHERE SERIALNAME IS NULL")
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SERIALNAME FROM TESTDB.TESTTABLE WHERE SERIALNAME IS NULL")
assert(actualResult equals List(Map("SERIALNAME" -> null)))
}
test("test the min function when null is included in string data type with dictionary_include"){
// See CARBONDATA-2152
- val actualResult = PrestoServer.executeQuery("SELECT MIN(SERIALNAME) FROM TESTDB.TESTTABLE")
+ val actualResult = prestoServer.executeQuery("SELECT MIN(SERIALNAME) FROM TESTDB.TESTTABLE")
val expectedResult = List(Map("_col0" -> "ASD14875"))
assert(actualResult.equals(expectedResult))
@@ -511,11 +511,11 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
test("test the show schemas result"){
- val actualResult = PrestoServer.executeQuery("SHOW SCHEMAS")
+ val actualResult = prestoServer.executeQuery("SHOW SCHEMAS")
assert(actualResult.equals(List(Map("Schema" -> "information_schema"), Map("Schema" -> "testdb"))))
}
test("test the show tables"){
- val actualResult = PrestoServer.executeQuery("SHOW TABLES")
+ val actualResult = prestoServer.executeQuery("SHOW TABLES")
assert(actualResult.equals(List(Map("Table" -> "testtable"))))
}
@@ -531,7 +531,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR operator on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT BONUS FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT BONUS FROM TESTDB.TESTTABLE WHERE" +
" BONUS < 600 OR BONUS > 5000 ORDER BY BONUS")
val expectedResult: List[Map[String, Any]] = List(
Map("BONUS" -> java.math.BigDecimal.valueOf(500.4140).setScale(4)),
@@ -544,7 +544,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the AND, OR operator on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" SHORTFIELD > 4 AND (SHORTFIELD < 10 or SHORTFIELD > 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 8),
@@ -553,7 +553,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR operator with multiple AND on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" (SHORTFIELD > 1 AND SHORTFIELD < 5) OR (SHORTFIELD > 10 AND SHORTFIELD < 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 4),
@@ -563,7 +563,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR, AND operator with on Different column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" ID < 7 AND (SHORTFIELD < 5 OR SHORTFIELD > 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 4),
@@ -572,7 +572,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the Timestamp greaterthan expression"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT DOB FROM TESTDB.TESTTABLE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT DOB FROM TESTDB.TESTTABLE" +
" WHERE DOB > timestamp '2016-01-01 00:00:00.0' order by DOB")
val expectedResult: List[Map[String, Any]] = List(
Map("DOB" -> new Timestamp(new java.util.Date(2016-1900,1-1,14,15,7,9).getTime)),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
new file mode 100644
index 0000000..4c2f51f
--- /dev/null
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.integrationtest
+
+import java.io.File
+import java.sql.SQLException
+import java.util
+
+import org.apache.commons.io.FileUtils
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.presto.server.PrestoServer
+import org.apache.carbondata.sdk.file.{CarbonWriter, Field, Schema}
+
+
+class PrestoTestNonTransactionalTableFiles extends FunSuiteLike with BeforeAndAfterAll {
+
+ private val logger = LogServiceFactory
+ .getLogService(classOf[PrestoTestNonTransactionalTableFiles].getCanonicalName)
+
+ private val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ private val storePath = s"$rootPath/integration/presto/target/store"
+ private val systemPath = s"$rootPath/integration/presto/target/system"
+ private val writerPath = storePath + "/sdk_output/files"
+ private val prestoServer = new PrestoServer
+
+ override def beforeAll: Unit = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
+ systemPath)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
+ "Presto")
+ prestoServer.startServer(storePath, "sdk_output")
+ }
+
+ override def afterAll(): Unit = {
+ prestoServer.stopServer()
+ }
+
+ def buildTestDataSingleFile(): Any = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ buildTestData(3, null)
+ }
+
+ def buildTestDataMultipleFiles(): Any = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ buildTestData(1000000, null)
+ }
+
+ def buildTestData(rows: Int, options: util.Map[String, String]): Any = {
+ buildTestData(rows, options, List("name"))
+ }
+
+ // prepare sdk writer output
+ def buildTestData(rows: Int,
+ options: util.Map[String, String],
+ sortColumns: List[String]): Any = {
+ val schema = new StringBuilder()
+ .append("[ \n")
+ .append(" {\"NaMe\":\"string\"},\n")
+ .append(" {\"age\":\"int\"},\n")
+ .append(" {\"height\":\"double\"}\n")
+ .append("]")
+ .toString()
+
+ try {
+ val builder = CarbonWriter.builder()
+ val writer =
+ if (options != null) {
+ builder.outputPath(writerPath)
+ .sortBy(sortColumns.toArray)
+ .uniqueIdentifier(
+ System.currentTimeMillis).withBlockSize(2).withLoadOptions(options)
+ .withCsvInput(Schema.parseJson(schema)).writtenBy("TestNonTransactionalCarbonTable")
+ .build()
+ } else {
+ builder.outputPath(writerPath)
+ .sortBy(sortColumns.toArray)
+ .uniqueIdentifier(
+ System.currentTimeMillis).withBlockSize(2)
+ .withCsvInput(Schema.parseJson(schema)).writtenBy("TestNonTransactionalCarbonTable")
+ .build()
+ }
+ var i = 0
+ while (i < rows) {
+ if ((options != null) && (i < 3)) {
+ // writing a bad record
+ writer.write(Array[String]("robot" + i, String.valueOf(i.toDouble / 2), "robot"))
+ } else {
+ writer
+ .write(Array[String]("robot" + i, String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ }
+ i += 1
+ }
+ if (options != null) {
+ //Keep one valid record. else carbon data file will not generate
+ writer.write(Array[String]("robot" + i, String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ }
+ writer.close()
+ } catch {
+ case ex: Throwable => throw new RuntimeException(ex)
+ }
+ }
+
+ // prepare sdk writer output with other schema
+ def buildTestDataOtherDataType(rows: Int, sortColumns: Array[String]): Any = {
+ val fields: Array[Field] = new Array[Field](3)
+ // same column name, but name as boolean type
+ fields(0) = new Field("name", DataTypes.BOOLEAN)
+ fields(1) = new Field("age", DataTypes.INT)
+ fields(2) = new Field("height", DataTypes.DOUBLE)
+
+ try {
+ val builder = CarbonWriter.builder()
+ val writer =
+ builder.outputPath(writerPath)
+ .uniqueIdentifier(System.currentTimeMillis()).withBlockSize(2).sortBy(sortColumns)
+ .withCsvInput(new Schema(fields)).writtenBy("TestNonTransactionalCarbonTable").build()
+ var i = 0
+ while (i < rows) {
+ writer.write(Array[String]("true", String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ i += 1
+ }
+ writer.close()
+ } catch {
+ case ex: Throwable => throw new RuntimeException(ex)
+ }
+ }
+
+ def cleanTestData(): Unit = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ }
+
+ def deleteFile(path: String, extension: String): Unit = {
+ val file: CarbonFile = FileFactory
+ .getCarbonFile(path, FileFactory.getFileType(path))
+
+ for (eachDir <- file.listFiles) {
+ if (!eachDir.isDirectory) {
+ if (eachDir.getName.endsWith(extension)) {
+ CarbonUtil.deleteFoldersAndFilesSilent(eachDir)
+ }
+ } else {
+ deleteFile(eachDir.getPath, extension)
+ }
+ }
+ }
+
+ test("test show schemas") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("show schemas ")
+ assert(actualResult
+ .equals(List(Map("Schema" -> "information_schema"),
+ Map("Schema" -> "sdk_output"),
+ Map("Schema" -> "testdb"))))
+ cleanTestData()
+ }
+
+ test("test show tables") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("show tables")
+ assert(actualResult
+ .equals(List(Map("Table" -> "files"))))
+ cleanTestData()
+ }
+
+ test("test read sdk output files") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT COUNT(*) AS RESULT FROM files ")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 3))
+ assert(actualResult.equals(expectedResult))
+ cleanTestData()
+ }
+
+ test("test read multiple carbondata and index files") {
+ buildTestDataMultipleFiles()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT COUNT(*) AS RESULT FROM files ")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 1000000))
+ assert(actualResult.equals(expectedResult))
+ cleanTestData()
+ }
+
+ test("test reading different schema") {
+ buildTestDataSingleFile()
+ buildTestDataOtherDataType(3, null)
+ val exception =
+ intercept[SQLException] {
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("select count(*) as RESULT from files ")
+ }
+ assert(exception.getMessage()
+ .contains("All the files doesn't have same schema"))
+ cleanTestData()
+ }
+
+ test("test reading without carbon index file") {
+ buildTestDataSingleFile()
+ deleteFile(writerPath, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT)
+ val exception =
+ intercept[SQLException] {
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("select * from files ")
+ }
+ assert(exception.getMessage()
+ .contains("No Index files are present in the table location"))
+ cleanTestData()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/db08fe1f/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index 1b1accf..5c0c40a 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -32,7 +32,6 @@ import com.facebook.presto.tests.DistributedQueryRunner
import com.google.common.collect.ImmutableMap
import org.slf4j.{Logger, LoggerFactory}
-import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.presto.CarbondataPlugin
class PrestoServer {
@@ -46,6 +45,7 @@ class PrestoServer {
val prestoProperties: util.Map[String, String] = Map(("http-server.http.port", "8086")).asJava
createSession
lazy val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
+ var dbName : String = null
/**
@@ -53,7 +53,24 @@ class PrestoServer {
*
* @param carbonStorePath the store path of carbon
*/
- def startServer(carbonStorePath: String) = {
+ def startServer(carbonStorePath: String): Unit = {
+
+ logger.info("======== STARTING PRESTO SERVER ========")
+ val queryRunner: DistributedQueryRunner = createQueryRunner(
+ prestoProperties, carbonStorePath)
+
+ logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator.getBaseUrl)
+ }
+
+ /**
+ * start the presto server
+ *
+ * @param carbonStorePath the store path of carbon
+ * @param dbName the database name , if not a default database
+ */
+ def startServer(carbonStorePath: String, dbName: String): Unit = {
+
+ this.dbName = dbName
logger.info("======== STARTING PRESTO SERVER ========")
val queryRunner: DistributedQueryRunner = createQueryRunner(
@@ -99,7 +116,7 @@ class PrestoServer {
def executeQuery(query: String): List[Map[String, Any]] = {
Try {
- val conn: Connection = createJdbcConnection
+ val conn: Connection = createJdbcConnection(dbName)
logger.info(s"***** executing the query ***** \n $query")
val statement = conn.createStatement()
val result: ResultSet = statement.executeQuery(query)
@@ -117,13 +134,17 @@ class PrestoServer {
*
* @return
*/
- private def createJdbcConnection: Connection = {
+ private def createJdbcConnection(dbName: String) = {
val JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"
- val DB_URL = "jdbc:presto://localhost:8086/carbondata/testdb"
-
+ var DB_URL : String = null
+ if (dbName == null) {
+ DB_URL = "jdbc:presto://localhost:8086/carbondata/default"
+ } else {
+ DB_URL = "jdbc:presto://localhost:8086/carbondata/" + dbName
+ }
val properties = new Properties
// The database Credentials
- properties.setProperty("user", "test");
+ properties.setProperty("user", "test")
// STEP 2: Register JDBC driver
Class.forName(JDBC_DRIVER)