You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/24 13:25:57 UTC
[01/50] [abbrv] carbondata git commit: Remove unnecessary mdk
computation code [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/fgdatamap 36c34cd02 -> 65d05375f (forced update)
Remove unnecessary mdk computation code
Remove unnecessary mdk computation code
This closes #1673
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/40c7e8ee
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/40c7e8ee
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/40c7e8ee
Branch: refs/heads/fgdatamap
Commit: 40c7e8ee89e4fb136752c33662e2798691f29f5a
Parents: 363731c
Author: Manhua <ke...@qq.com>
Authored: Sat Dec 16 11:34:07 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Dec 16 23:17:10 2017 +0800
----------------------------------------------------------------------
.../core/datastore/page/key/TablePageKey.java | 14 +++++---------
.../carbondata/core/util/CarbonMetadataUtilTest.java | 2 +-
.../apache/carbondata/processing/store/TablePage.java | 3 +--
3 files changed, 7 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/40c7e8ee/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index f8a10ca..244d8fc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -23,8 +23,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.NonDictionaryUtil;
public class TablePageKey {
@@ -50,30 +48,28 @@ public class TablePageKey {
// endkey for no dictionary columns after packing into one column
private byte[] packedNoDictEndKey;
- private KeyGenerator mdkGenerator;
private SegmentProperties segmentProperties;
private boolean hasNoDictionary;
- public TablePageKey(int pageSize, KeyGenerator mdkGenerator, SegmentProperties segmentProperties,
- boolean hasNoDictionary) {
+ public TablePageKey(int pageSize, SegmentProperties segmentProperties,
+ boolean hasNoDictionary) {
this.pageSize = pageSize;
- this.mdkGenerator = mdkGenerator;
this.segmentProperties = segmentProperties;
this.hasNoDictionary = hasNoDictionary;
}
/** update all keys based on the input row */
- public void update(int rowId, CarbonRow row, byte[] mdk) throws KeyGenException {
+ public void update(int rowId, CarbonRow row, byte[] mdk) {
if (hasNoDictionary) {
currentNoDictionaryKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
}
if (rowId == 0) {
- startKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
+ startKey = mdk;
noDictStartKey = currentNoDictionaryKey;
}
noDictEndKey = currentNoDictionaryKey;
if (rowId == pageSize - 1) {
- endKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
+ endKey = mdk;
finalizeKeys();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/40c7e8ee/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 4cf3d11..28cfcf2 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -209,7 +209,7 @@ public class CarbonMetadataUtilTest {
}
};
- TablePageKey key = new TablePageKey(3, null, segmentProperties, false);
+ TablePageKey key = new TablePageKey(3, segmentProperties, false);
EncodedTablePage encodedTablePage = EncodedTablePage.newInstance(3, new EncodedColumnPage[0], new EncodedColumnPage[0],
key);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/40c7e8ee/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 6a9aba1..ff33823 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -124,8 +124,7 @@ public class TablePage {
measurePages[i] = page;
}
boolean hasNoDictionary = noDictDimensionPages.length > 0;
- this.key = new TablePageKey(pageSize, model.getMDKeyGenerator(), model.getSegmentProperties(),
- hasNoDictionary);
+ this.key = new TablePageKey(pageSize, model.getSegmentProperties(), hasNoDictionary);
// for complex type, `complexIndexMap` is used in multithread (in multiple Producer),
// we need to clone the index map to make it thread safe
[15/50] [abbrv] carbondata git commit: [CARBONDATA-1743] fix
conurrent pre-agg creation and query
Posted by ja...@apache.org.
[CARBONDATA-1743] fix conurrent pre-agg creation and query
When a preaggregate table is created and load is in progress the aggregate query on main table tries to get the data from pre-aggregate table which used to give empty results.If the size of preaggregate table is 0 meaning no data is loaded then hit the main table instead of aggregate table
This closes #1521
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e6497e14
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e6497e14
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e6497e14
Branch: refs/heads/fgdatamap
Commit: e6497e14e6454f0990354f542e0e7f2d888b9725
Parents: 804ddb7
Author: kunal642 <ku...@gmail.com>
Authored: Fri Nov 17 12:13:25 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Tue Dec 19 19:29:03 2017 +0530
----------------------------------------------------------------------
.../TestPreAggregateTableSelection.scala | 17 ++++-----
.../TestTimeseriesTableSelection.scala | 32 +++++++++--------
.../sql/hive/CarbonPreAggregateRules.scala | 38 ++++++++++++--------
.../apache/spark/sql/hive/CarbonRelation.scala | 19 ++++++----
4 files changed, 63 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6497e14/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 5b36826..7157b8a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -27,7 +27,14 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
sql("drop table if exists mainTable")
- sql("drop table if exists lineitem")
+ sql("drop table if exists agg0")
+ sql("drop table if exists agg1")
+ sql("drop table if exists agg2")
+ sql("drop table if exists agg3")
+ sql("drop table if exists agg4")
+ sql("drop table if exists agg5")
+ sql("drop table if exists agg6")
+ sql("drop table if exists agg7")
sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
sql("create datamap agg0 on table mainTable using 'preaggregate' as select name from mainTable group by name")
sql("create datamap agg1 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name")
@@ -39,8 +46,6 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
sql("create datamap agg7 on table mainTable using 'preaggregate' as select name,max(age) from mainTable group by name")
sql("create datamap agg8 on table maintable using 'preaggregate' as select name, sum(id), avg(id) from maintable group by name")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- sql("create table if not exists lineitem(L_SHIPDATE string,L_SHIPMODE string,L_SHIPINSTRUCT string,L_RETURNFLAG string,L_RECEIPTDATE string,L_ORDERKEY string,L_PARTKEY string,L_SUPPKEY string,L_LINENUMBER int,L_QUANTITY double,L_EXTENDEDPRICE double,L_DISCOUNT double,L_TAX double,L_LINESTATUS string,L_COMMITDATE string,L_COMMENT string) STORED BY 'org.apache.carbondata.format'TBLPROPERTIES ('table_blocksize'='128','NO_INVERTED_INDEX'='L_SHIPDATE,L_SHIPMODE,L_SHIPINSTRUCT,L_RETURNFLAG,L_RECEIPTDATE,L_ORDERKEY,L_PARTKEY,L_SUPPKEY','sort_columns'='')")
- sql("create datamap agr_lineitem ON TABLE lineitem USING 'preaggregate' as select L_RETURNFLAG,L_LINESTATUS,sum (L_QUANTITY),sum(L_EXTENDEDPRICE) from lineitem group by L_RETURNFLAG, L_LINESTATUS")
}
test("test sum and avg on same column should give proper results") {
@@ -138,11 +143,7 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
val df = sql("select count(id) from mainTable")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg3")
}
-
- test("test PreAggregate table selection 19") {
- val df = sql("select L_RETURNFLAG,L_LINESTATUS,sum(L_QUANTITY),sum(L_EXTENDEDPRICE) from lineitem group by L_RETURNFLAG, L_LINESTATUS")
- preAggTableValidator(df.queryExecution.analyzed, "lineitem_agr_lineitem")
- }
+
test("test PreAggregate table selection 20") {
val df = sql("select name from mainTable group by name order by name")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6497e14/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
index 0990f87..725ac24 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
@@ -28,71 +28,73 @@ class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
sql("drop table if exists mainTable")
- sql("CREATE TABLE mainTable(dataTime timestamp, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select dataTime, sum(age) from mainTable group by dataTime")
+ sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
+ sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
}
+
test("test PreAggregate table selection 1") {
- val df = sql("select dataTime from mainTable group by dataTime")
+ val df = sql("select mytime from mainTable group by mytime")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
test("test PreAggregate table selection 2") {
- val df = sql("select timeseries(dataTime,'hour') from mainTable group by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour') from mainTable group by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
test("test PreAggregate table selection 3") {
- val df = sql("select timeseries(dataTime,'milli') from mainTable group by timeseries(dataTime,'milli')")
+ val df = sql("select timeseries(mytime,'milli') from mainTable group by timeseries(mytime,'milli')")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
test("test PreAggregate table selection 4") {
- val df = sql("select timeseries(dataTime,'year') from mainTable group by timeseries(dataTime,'year')")
+ val df = sql("select timeseries(mytime,'year') from mainTable group by timeseries(mytime,'year')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_year")
}
test("test PreAggregate table selection 5") {
- val df = sql("select timeseries(dataTime,'day') from mainTable group by timeseries(dataTime,'day')")
+ val df = sql("select timeseries(mytime,'day') from mainTable group by timeseries(mytime,'day')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_day")
}
test("test PreAggregate table selection 6") {
- val df = sql("select timeseries(dataTime,'month') from mainTable group by timeseries(dataTime,'month')")
+ val df = sql("select timeseries(mytime,'month') from mainTable group by timeseries(mytime,'month')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_month")
}
test("test PreAggregate table selection 7") {
- val df = sql("select timeseries(dataTime,'minute') from mainTable group by timeseries(dataTime,'minute')")
+ val df = sql("select timeseries(mytime,'minute') from mainTable group by timeseries(mytime,'minute')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_minute")
}
test("test PreAggregate table selection 8") {
- val df = sql("select timeseries(dataTime,'second') from mainTable group by timeseries(dataTime,'second')")
+ val df = sql("select timeseries(mytime,'second') from mainTable group by timeseries(mytime,'second')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_second")
}
test("test PreAggregate table selection 9") {
- val df = sql("select timeseries(dataTime,'hour') from mainTable where timeseries(dataTime,'hour')='x' group by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour') from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
}
test("test PreAggregate table selection 10") {
- val df = sql("select timeseries(dataTime,'hour') from mainTable where timeseries(dataTime,'hour')='x' group by timeseries(dataTime,'hour') order by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour') from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
}
test("test PreAggregate table selection 11") {
- val df = sql("select timeseries(dataTime,'hour'),sum(age) from mainTable where timeseries(dataTime,'hour')='x' group by timeseries(dataTime,'hour') order by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour'),sum(age) from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
}
test("test PreAggregate table selection 12") {
- val df = sql("select timeseries(dataTime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(dataTime,'hour')='x' group by timeseries(dataTime,'hour') order by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
}
test("test PreAggregate table selection 13") {
- val df = sql("select timeseries(dataTime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(dataTime,'hour')='x' and name='vishal' group by timeseries(dataTime,'hour') order by timeseries(dataTime,'hour')")
+ val df = sql("select timeseries(mytime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(mytime,'hour')='x' and name='vishal' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed,"maintable")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6497e14/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index c28cc44..76c39a4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -296,21 +296,31 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
val selectedDataMapSchemas = aggregateTableSelector.selectPreAggDataMapSchema()
// if it does not match with any pre aggregate table return the same plan
if (!selectedDataMapSchemas.isEmpty) {
- // sort the selected child schema based on size to select smallest pre aggregate table
+ // filter the selected child schema based on size to select the pre-aggregate tables
+ // that are nonEmpty
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val (aggDataMapSchema, carbonRelation, relation) =
- selectedDataMapSchemas.asScala.map { selectedDataMapSchema =>
- val identifier = TableIdentifier(
- selectedDataMapSchema.getRelationIdentifier.getTableName,
- Some(selectedDataMapSchema.getRelationIdentifier.getDatabaseName))
- val carbonRelation =
- catalog.lookupRelation(identifier)(sparkSession).asInstanceOf[CarbonRelation]
- val relation = sparkSession.sessionState.catalog.lookupRelation(identifier)
- (selectedDataMapSchema, carbonRelation, relation)
- }.minBy(f => f._2.sizeInBytes)
- val newRelation = new FindDataSourceTable(sparkSession).apply(relation)
- // transform the query plan based on selected child schema
- transformPreAggQueryPlan(plan, aggDataMapSchema, newRelation)
+ val relationBuffer = selectedDataMapSchemas.asScala.map { selectedDataMapSchema =>
+ val identifier = TableIdentifier(
+ selectedDataMapSchema.getRelationIdentifier.getTableName,
+ Some(selectedDataMapSchema.getRelationIdentifier.getDatabaseName))
+ val carbonRelation =
+ catalog.lookupRelation(identifier)(sparkSession).asInstanceOf[CarbonRelation]
+ val relation = sparkSession.sessionState.catalog.lookupRelation(identifier)
+ (selectedDataMapSchema, carbonRelation, relation)
+ }.filter(_._2.sizeInBytes != 0L)
+ if (relationBuffer.isEmpty) {
+ // If the size of relation Buffer is 0 then it means that none of the pre-aggregate
+ // tables have date yet.
+ // In this case we would return the original plan so that the query hits the parent
+ // table.
+ plan
+ } else {
+ // If the relationBuffer is nonEmpty then find the table with the minimum size.
+ val (aggDataMapSchema, _, relation) = relationBuffer.minBy(_._2.sizeInBytes)
+ val newRelation = new FindDataSourceTable(sparkSession).apply(relation)
+ // transform the query plan based on selected child schema
+ transformPreAggQueryPlan(plan, aggDataMapSchema, newRelation)
+ }
} else {
plan
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6497e14/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index aadce98..87be2d2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -22,6 +22,7 @@ import scala.Array.canBuildFrom
import scala.collection.JavaConverters._
import scala.util.parsing.combinator.RegexParsers
+import org.apache.spark.sql.CarbonEnv
import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
@@ -205,12 +206,18 @@ case class CarbonRelation(
carbonTable.getAbsoluteTableIdentifier)
if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
- val tablePath = CarbonStorePath.getCarbonTablePath(
- carbonTable.getAbsoluteTableIdentifier).getPath
- val fileType = FileFactory.getFileType(tablePath)
- if(FileFactory.isFileExist(tablePath, fileType)) {
- tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
- sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
+ if (new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
+ .getValidAndInvalidSegments.getValidSegments.isEmpty) {
+ sizeInBytesLocalValue = 0L
+ } else {
+ val tablePath = CarbonStorePath.getCarbonTablePath(
+ carbonTable.getTablePath,
+ carbonTable.getCarbonTableIdentifier).getPath
+ val fileType = FileFactory.getFileType(tablePath)
+ if (FileFactory.isFileExist(tablePath, fileType)) {
+ tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
+ sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
+ }
}
}
sizeInBytesLocalValue
[40/50] [abbrv] carbondata git commit: [CARBONDATA-1920]
[PrestoIntegration] Sparksql query result is not same as presto on same sql
Posted by ja...@apache.org.
[CARBONDATA-1920] [PrestoIntegration] Sparksql query result is not same as presto on same sql
In Stream Readers we are only decoding dictionary value in case our javatype is slice and it is not decimal type
This closes #1695
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6add19ad
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6add19ad
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6add19ad
Branch: refs/heads/fgdatamap
Commit: 6add19adee6f061d1399525b113eea11d1085713
Parents: dfe3097
Author: anubhav100 <an...@knoldus.in>
Authored: Tue Dec 5 12:25:58 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Dec 22 14:11:52 2017 +0800
----------------------------------------------------------------------
.../carbondata/presto/CarbondataPageSource.java | 20 +--
.../carbondata/presto/PrestoFilterUtil.java | 51 +++-----
.../readers/DecimalSliceStreamReader.java | 87 +++++++++----
.../presto/readers/DoubleStreamReader.java | 39 ++++--
.../presto/readers/IntegerStreamReader.java | 54 +++++---
.../presto/readers/LongStreamReader.java | 38 ++++--
.../presto/readers/ShortStreamReader.java | 51 +++++---
.../presto/readers/SliceStreamReader.java | 31 ++---
.../presto/readers/StreamReaders.java | 74 +++++++----
.../CarbonDictionaryDecodeReadSupport.scala | 40 +++---
.../presto/util/CarbonDataStoreCreator.scala | 126 ++++++++++---------
11 files changed, 366 insertions(+), 245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index e8ecba3..1679f29 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -65,7 +65,8 @@ class CarbondataPageSource implements ConnectorPageSource {
private long nanoEnd;
CarbondataPageSource(RecordSet recordSet) {
- this(requireNonNull(recordSet, "recordSet is null").getColumnTypes(), recordSet.cursor());
+ this(requireNonNull(recordSet, "recordSet is null").getColumnTypes(),
+ recordSet.cursor());
}
private CarbondataPageSource(List<Type> types, RecordCursor cursor) {
@@ -109,7 +110,6 @@ class CarbondataPageSource implements ConnectorPageSource {
return null;
}
}
-
} else {
close();
return null;
@@ -134,17 +134,10 @@ class CarbondataPageSource implements ConnectorPageSource {
closeWithSuppression(e);
throw e;
}
- catch ( RuntimeException e) {
- closeWithSuppression(e);
- throw new CarbonDataLoadingException("Exception when creating the Carbon data Block", e);
- } catch (InterruptedException e) {
- closeWithSuppression(e);
- throw new CarbonDataLoadingException("Exception when creating the Carbon data Block", e);
- } catch (IOException e) {
+ catch ( RuntimeException | InterruptedException | IOException e) {
closeWithSuppression(e);
throw new CarbonDataLoadingException("Exception when creating the Carbon data Block", e);
}
-
}
@Override public long getSystemMemoryUsage() {
@@ -205,9 +198,7 @@ class CarbondataPageSource implements ConnectorPageSource {
if (loaded) {
return;
}
-
checkState(batchId == expectedBatchId);
-
try {
Block block = readers[columnIndex].readBlock(type);
lazyBlock.setBlock(block);
@@ -215,7 +206,6 @@ class CarbondataPageSource implements ConnectorPageSource {
catch (IOException e) {
throw new CarbonDataLoadingException("Error in Reading Data from Carbondata ", e);
}
-
loaded = true;
}
}
@@ -231,8 +221,8 @@ class CarbondataPageSource implements ConnectorPageSource {
requireNonNull(types);
StreamReader[] readers = new StreamReader[types.size()];
for (int i = 0; i < types.size(); i++) {
- readers[i] =
- StreamReaders.createStreamReader(types.get(i), readSupport.getSliceArrayBlock(i));
+ readers[i] = StreamReaders.createStreamReader(types.get(i), readSupport
+ .getSliceArrayBlock(i),readSupport.getDictionaries()[i]);
}
return readers;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
index 31d5ba6..a69013f 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -125,20 +125,14 @@ public class PrestoFilterUtil {
} else {
GreaterThanExpression greater = new GreaterThanExpression(colExpression,
new LiteralExpression(value, coltype));
- if(valueExpressionMap.get(value) == null) {
- valueExpressionMap.put(value, new ArrayList<>());
- }
- valueExpressionMap.get(value).add(greater);
+ valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
}
break;
case EXACTLY:
GreaterThanEqualToExpression greater =
new GreaterThanEqualToExpression(colExpression,
new LiteralExpression(value, coltype));
- if(valueExpressionMap.get(value) == null) {
- valueExpressionMap.put(value, new ArrayList<>());
- }
- valueExpressionMap.get(value).add(greater);
+ valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
@@ -154,18 +148,12 @@ public class PrestoFilterUtil {
case EXACTLY:
LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression,
new LiteralExpression(value, coltype));
- if(valueExpressionMap.get(value) == null) {
- valueExpressionMap.put(value, new ArrayList<>());
- }
- valueExpressionMap.get(value).add(less);
+ valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less);
break;
case BELOW:
LessThanExpression less2 =
new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
- if(valueExpressionMap.get(value) == null) {
- valueExpressionMap.put(value, new ArrayList<>());
- }
- valueExpressionMap.get(value).add(less2);
+ valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less2);
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
@@ -208,14 +196,14 @@ public class PrestoFilterUtil {
valuefilters.add(finalFilters);
}
- if(valuefilters.size() == 1){
+ if (valuefilters.size() == 1) {
finalFilters = valuefilters.get(0);
} else if (valuefilters.size() >= 2) {
- finalFilters = new AndExpression(valuefilters.get(0), valuefilters.get(1));
- for (int i = 2; i < valuefilters.size() ; i++) {
- finalFilters = new AndExpression(finalFilters, valuefilters.get(i));
- }
- }
+ finalFilters = new AndExpression(valuefilters.get(0), valuefilters.get(1));
+ for (int i = 2; i < valuefilters.size(); i++) {
+ finalFilters = new AndExpression(finalFilters, valuefilters.get(i));
+ }
+ }
filters.add(finalFilters);
}
@@ -236,8 +224,9 @@ public class PrestoFilterUtil {
}
private static Object ConvertDataByType(Object rawdata, Type type) {
- if (type.equals(IntegerType.INTEGER)) return Integer.valueOf(rawdata.toString());
- // new Integer((rawdata.toString()));
+ if (type.equals(IntegerType.INTEGER) || type.equals(SmallintType.SMALLINT))
+ return Integer.valueOf(rawdata.toString());
+ // new Integer((rawdata.toString()));
else if (type.equals(BigintType.BIGINT)) return rawdata;
else if (type.equals(VarcharType.VARCHAR)) {
if (rawdata instanceof Slice) {
@@ -253,18 +242,18 @@ public class PrestoFilterUtil {
c.add(Calendar.DAY_OF_YEAR, ((Long) rawdata).intValue());
Date date = c.getTime();
return date.getTime() * 1000;
- }
- else if (type instanceof DecimalType) {
- if(rawdata instanceof Double) {
+ } else if (type instanceof DecimalType) {
+ if (rawdata instanceof Double) {
return new BigDecimal((Double) rawdata);
- } else if (rawdata instanceof Long) {
+ } else if (rawdata instanceof Long) {
return new BigDecimal(new BigInteger(String.valueOf(rawdata)),
((DecimalType) type).getScale());
- } else if(rawdata instanceof Slice) {
- return new BigDecimal(Decimals.decodeUnscaledValue((Slice) rawdata), ((DecimalType) type).getScale());
+ } else if (rawdata instanceof Slice) {
+ return new BigDecimal(Decimals.decodeUnscaledValue((Slice) rawdata),
+ ((DecimalType) type).getScale());
}
} else if (type.equals(TimestampType.TIMESTAMP)) {
- return (Long)rawdata * 1000;
+ return (Long) rawdata * 1000;
}
return rawdata;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
index ad798d4..e6ac386 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
@@ -20,6 +20,11 @@ package org.apache.carbondata.presto.readers;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.util.Objects;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
@@ -42,6 +47,9 @@ import static java.math.RoundingMode.HALF_UP;
*/
public class DecimalSliceStreamReader extends AbstractStreamReader {
+ private Dictionary dictionary;
+ private boolean isDictionary;
+
private final char[] buffer = new char[100];
@@ -49,6 +57,11 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
}
+ public DecimalSliceStreamReader(boolean isDictionary, Dictionary dictionary) {
+ this.dictionary = dictionary;
+ this.isDictionary = isDictionary;
+ }
+
/**
* Create Block for DecimalType
* @param type
@@ -63,20 +76,18 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
if(isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
- int scale = ((DecimalType)type).getScale();
- int precision = ((DecimalType)type).getPrecision();
if (columnVector != null) {
if(columnVector.anyNullsSet())
{
- handleNullInVector(type, numberOfRows, builder, scale, precision);
+ handleNullInVector(type, numberOfRows, builder);
} else {
if(isShortDecimal(type)) {
- populateShortDecimalVector(type, numberOfRows, builder, scale, precision);
+ populateShortDecimalVector(type, numberOfRows, builder);
} else {
- populateLongDecimalVector(type, numberOfRows, builder, scale, precision);
+ populateLongDecimalVector(type, numberOfRows, builder);
}
}
- }
+ }
} else {
if (streamData != null) {
@@ -180,8 +191,7 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
}
- private void handleNullInVector(Type type, int numberOfRows, BlockBuilder builder, int scale,
- int precision) {
+ private void handleNullInVector(Type type, int numberOfRows, BlockBuilder builder) {
for (int i = 0; i < numberOfRows; i++) {
if (columnVector.isNullAt(i)) {
builder.appendNull();
@@ -189,32 +199,63 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
if (isShortDecimal(type)) {
BigDecimal decimalValue = (BigDecimal)columnVector.getData(i);
long rescaledDecimal = Decimals.rescale(decimalValue.unscaledValue().longValue(),
- decimalValue.scale(), scale);
+ decimalValue.scale(),((DecimalType) type).getScale());
type.writeLong(builder, rescaledDecimal);
} else {
- Slice slice =
- getSlice(columnVector.getData(i), type);
+ Slice slice = getSlice(columnVector.getData(i), type);
type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
}
}
}
}
- private void populateShortDecimalVector(Type type, int numberOfRows, BlockBuilder builder,
- int scale, int precision) {
- for (int i = 0; i < numberOfRows; i++) {
- BigDecimal decimalValue = (BigDecimal)columnVector.getData(i);
- long rescaledDecimal = Decimals.rescale(decimalValue.unscaledValue().longValue(),
- decimalValue.scale(), scale);
- type.writeLong(builder, rescaledDecimal);
+ private void populateShortDecimalVector(Type type, int numberOfRows, BlockBuilder builder) {
+ DecimalType decimalType = (DecimalType) type;
+
+ if (isDictionary) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int)columnVector.getData(i);
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
+ if(Objects.isNull(data)) {
+ builder.appendNull();
+ } else {
+ BigDecimal decimalValue = (BigDecimal) data;
+ long rescaledDecimal =
+ Decimals.rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),decimalType.getScale());
+ type.writeLong(builder, rescaledDecimal);
+ }
+ }
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ BigDecimal decimalValue = (BigDecimal) columnVector.getData(i);
+ long rescaledDecimal =
+ Decimals.rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),decimalType.getScale());
+ type.writeLong(builder, rescaledDecimal);
+ }
}
}
- private void populateLongDecimalVector(Type type, int numberOfRows, BlockBuilder builder,
- int scale, int precision) {
- for (int i = 0; i < numberOfRows; i++) {
- Slice slice = getSlice((BigDecimal)columnVector.getData(i), type);
- type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
+ private void populateLongDecimalVector(Type type, int numberOfRows, BlockBuilder builder) {
+ if (isDictionary) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ DecimalType decimalType = (DecimalType) type;
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
+ if(Objects.isNull(data)) {
+ builder.appendNull();
+ } else {
+ BigDecimal decimalValue = (BigDecimal) data;
+ Slice slice = getSlice(decimalValue, type);
+ type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
+ }
+ }
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ Slice slice = getSlice((columnVector.getData(i)), type);
+ type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
index a1910b7..7968ae6 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
@@ -19,6 +19,10 @@ package org.apache.carbondata.presto.readers;
import java.io.IOException;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
@@ -29,10 +33,18 @@ import com.facebook.presto.spi.type.Type;
*/
public class DoubleStreamReader extends AbstractStreamReader {
+ private boolean isDictionary;
+ private Dictionary dictionary;
+
public DoubleStreamReader() {
}
+ public DoubleStreamReader(boolean isDictionary, Dictionary dictionary) {
+ this.isDictionary = isDictionary;
+ this.dictionary = dictionary;
+ }
+
/**
* Create the DoubleType Block
*
@@ -47,10 +59,9 @@ public class DoubleStreamReader extends AbstractStreamReader {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(columnVector.anyNullsSet()) {
+ if (columnVector.anyNullsSet()) {
handleNullInVector(type, numberOfRows, builder);
- }
- else {
+ } else {
populateVector(type, numberOfRows, builder);
}
}
@@ -72,15 +83,29 @@ public class DoubleStreamReader extends AbstractStreamReader {
if (columnVector.isNullAt(i)) {
builder.appendNull();
} else {
- type.writeDouble(builder, (Double)columnVector.getData(i));
+ type.writeDouble(builder, (Double) columnVector.getData(i));
}
}
}
private void populateVector(Type type, int numberOfRows, BlockBuilder builder) {
- for (int i = 0; i < numberOfRows; i++) {
- type.writeDouble(builder, (Double)columnVector.getData(i));
+ if (isDictionary) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.DOUBLE);
+ if (data != null) {
+ type.writeDouble(builder, (Double) data);
+ } else {
+ builder.appendNull();
+ }
+
+ }
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeDouble(builder, (Double) columnVector.getData(i));
+ }
}
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/IntegerStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/IntegerStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/IntegerStreamReader.java
index 33fc529..5b20925 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/IntegerStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/IntegerStreamReader.java
@@ -19,6 +19,10 @@ package org.apache.carbondata.presto.readers;
import java.io.IOException;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
@@ -26,38 +30,40 @@ import com.facebook.presto.spi.type.Type;
public class IntegerStreamReader extends AbstractStreamReader {
+ private Dictionary dictionaryValues;
+ private boolean isDictionary;
+
+ public IntegerStreamReader() {
- public IntegerStreamReader( ) {
+ }
+ public IntegerStreamReader(boolean isDictionary, Dictionary dictionary) {
+ this.dictionaryValues = dictionary;
+ this.isDictionary = isDictionary;
}
- public Block readBlock(Type type)
- throws IOException
- {
- int numberOfRows = 0;
- BlockBuilder builder = null;
- if(isVectorReader) {
+ public Block readBlock(Type type) throws IOException {
+ int numberOfRows;
+ BlockBuilder builder;
+ if (isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(columnVector.anyNullsSet()) {
+ if (columnVector.anyNullsSet()) {
handleNullInVector(type, numberOfRows, builder);
- }
- else {
+ } else {
populateVector(type, numberOfRows, builder);
}
}
-
} else {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (streamData != null) {
- for(int i = 0; i < numberOfRows ; i++ ){
- type.writeLong(builder, ((Integer)streamData[i]).longValue());
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeLong(builder, ((Integer) streamData[i]).longValue());
}
}
}
-
return builder.build();
}
@@ -72,9 +78,23 @@ public class IntegerStreamReader extends AbstractStreamReader {
}
private void populateVector(Type type, int numberOfRows, BlockBuilder builder) {
- for (int i = 0; i < numberOfRows; i++) {
- type.writeLong(builder, (Integer) columnVector.getData(i));
+ if (isDictionary) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionaryValues.getDictionaryValueForKey(value),
+ DataTypes.INT);
+ if (data != null) {
+ type.writeLong(builder, ((Integer) data).longValue());
+ } else {
+ builder.appendNull();
+ }
+ }
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ Integer value = (Integer) columnVector.getData(i);
+ type.writeLong(builder, value.longValue());
}
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/LongStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/LongStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/LongStreamReader.java
index d7ccda0..ab68889 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/LongStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/LongStreamReader.java
@@ -19,6 +19,10 @@ package org.apache.carbondata.presto.readers;
import java.io.IOException;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
@@ -26,25 +30,31 @@ import com.facebook.presto.spi.type.Type;
public class LongStreamReader extends AbstractStreamReader {
+ private boolean isDictionary;
+ private Dictionary dictionary;
+
public LongStreamReader() {
}
+ public LongStreamReader(boolean isDictionary, Dictionary dictionary) {
+ this.isDictionary = isDictionary;
+ this.dictionary = dictionary;
+ }
+
public Block readBlock(Type type) throws IOException {
- int numberOfRows = 0;
- BlockBuilder builder = null;
+ int numberOfRows;
+ BlockBuilder builder;
if (isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(columnVector.anyNullsSet()) {
+ if (columnVector.anyNullsSet()) {
handleNullInVector(type, numberOfRows, builder);
- }
- else {
+ } else {
populateVector(type, numberOfRows, builder);
}
}
-
} else {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
@@ -54,7 +64,6 @@ public class LongStreamReader extends AbstractStreamReader {
}
}
}
-
return builder.build();
}
@@ -63,14 +72,25 @@ public class LongStreamReader extends AbstractStreamReader {
if (columnVector.isNullAt(i)) {
builder.appendNull();
} else {
- type.writeLong(builder, (Long)columnVector.getData(i));
+ type.writeLong(builder, (Long) columnVector.getData(i));
}
}
}
private void populateVector(Type type, int numberOfRows, BlockBuilder builder) {
for (int i = 0; i < numberOfRows; i++) {
- type.writeLong(builder, (Long)columnVector.getData(i));
+ if (isDictionary) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.LONG);
+ if (data != null) {
+ type.writeLong(builder, (Long) data);
+ } else {
+ builder.appendNull();
+ }
+ } else {
+ type.writeLong(builder, (long) columnVector.getData(i));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ShortStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ShortStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ShortStreamReader.java
index 87ebb12..4a28761 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ShortStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ShortStreamReader.java
@@ -19,6 +19,10 @@ package org.apache.carbondata.presto.readers;
import java.io.IOException;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
@@ -26,24 +30,28 @@ import com.facebook.presto.spi.type.Type;
public class ShortStreamReader extends AbstractStreamReader {
+ private boolean isDictionary;
+ private Dictionary dictionary;
+
+ public ShortStreamReader() {
- public ShortStreamReader( ) {
+ }
+ public ShortStreamReader(boolean isDictionary, Dictionary dictionary) {
+ this.isDictionary = isDictionary;
+ this.dictionary = dictionary;
}
- public Block readBlock(Type type)
- throws IOException
- {
- int numberOfRows = 0;
- BlockBuilder builder = null;
- if(isVectorReader) {
+ public Block readBlock(Type type) throws IOException {
+ int numberOfRows;
+ BlockBuilder builder;
+ if (isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(columnVector.anyNullsSet()) {
+ if (columnVector.anyNullsSet()) {
handleNullInVector(type, numberOfRows, builder);
- }
- else {
+ } else {
populateVector(type, numberOfRows, builder);
}
}
@@ -52,8 +60,8 @@ public class ShortStreamReader extends AbstractStreamReader {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (streamData != null) {
- for(int i = 0; i < numberOfRows ; i++ ){
- type.writeLong(builder, (short) streamData[i]);
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeLong(builder, (Short) streamData[i]);
}
}
}
@@ -66,15 +74,28 @@ public class ShortStreamReader extends AbstractStreamReader {
if (columnVector.isNullAt(i)) {
builder.appendNull();
} else {
- type.writeLong(builder, ((short) columnVector.getData(i)));
+ type.writeLong(builder, ((Short) columnVector.getData(i)));
}
}
}
private void populateVector(Type type, int numberOfRows, BlockBuilder builder) {
- for (int i = 0; i < numberOfRows; i++) {
- type.writeLong(builder, ((short) columnVector.getData(i)));
+ if (isDictionary) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil
+ .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.SHORT);
+ if (data != null) {
+ type.writeLong(builder, (Short) data);
+ } else {
+ builder.appendNull();
+ }
+ }
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeLong(builder, (Short) columnVector.getData(i));
}
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
index f307331..00b4bce 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
@@ -34,43 +34,40 @@ import static io.airlift.slice.Slices.wrappedBuffer;
*/
public class SliceStreamReader extends AbstractStreamReader {
-
private boolean isDictionary;
- private SliceArrayBlock dictionaryBlock;
+ private SliceArrayBlock dictionarySliceArrayBlock;
- public SliceStreamReader() {}
+ public SliceStreamReader() {
+ }
- public SliceStreamReader(boolean isDictionary, SliceArrayBlock dictionaryBlock) {
+ public SliceStreamReader(boolean isDictionary, SliceArrayBlock dictionarySliceArrayBlock) {
this.isDictionary = isDictionary;
- this.dictionaryBlock = dictionaryBlock;
+ this.dictionarySliceArrayBlock = dictionarySliceArrayBlock;
}
/**
* Function to create the Slice Block
+ *
* @param type
* @return
* @throws IOException
*/
- public Block readBlock(Type type)
- throws IOException
- {
- int numberOfRows = 0;
- BlockBuilder builder = null;
- if(isVectorReader) {
+ public Block readBlock(Type type) throws IOException {
+ int numberOfRows;
+ BlockBuilder builder;
+ if (isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(isDictionary) {
+ if (isDictionary) {
int[] values = new int[numberOfRows];
for (int i = 0; i < numberOfRows; i++) {
if (!columnVector.isNullAt(i)) {
values[i] = (Integer) columnVector.getData(i);
}
}
- Block block = new DictionaryBlock(batchSize, dictionaryBlock, values);
-
- return block;
+ return new DictionaryBlock(batchSize, dictionarySliceArrayBlock, values);
} else {
for (int i = 0; i < numberOfRows; i++) {
if (columnVector.isNullAt(i)) {
@@ -85,15 +82,13 @@ public class SliceStreamReader extends AbstractStreamReader {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (streamData != null) {
- for(int i = 0; i < numberOfRows ; i++ ){
+ for (int i = 0; i < numberOfRows; i++) {
type.writeSlice(builder, utf8Slice(streamData[i].toString()));
}
}
}
return builder.build();
-
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
index df1f8d6..b9b8b7e 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.presto.readers;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+
import com.facebook.presto.spi.block.SliceArrayBlock;
import com.facebook.presto.spi.type.DateType;
import com.facebook.presto.spi.type.DecimalType;
@@ -32,39 +34,61 @@ import io.airlift.slice.Slice;
public final class StreamReaders {
/**
* This function select Stream readers based on Type and use it.
+ *
* @param type
- * @param dictionary
+ * @param dictionarySliceArrayBlock
* @return StreamReader
*/
- public static StreamReader createStreamReader(Type type, SliceArrayBlock dictionary) {
+ public static StreamReader createStreamReader(Type type,
+ SliceArrayBlock dictionarySliceArrayBlock, Dictionary dictionary) {
Class<?> javaType = type.getJavaType();
- if (javaType == long.class) {
- if(type instanceof IntegerType || type instanceof DateType) {
- return new IntegerStreamReader();
- } else if (type instanceof DecimalType) {
- return new DecimalSliceStreamReader();
- } else if (type instanceof SmallintType) {
- return new ShortStreamReader();
- } else if (type instanceof TimestampType) {
- return new TimestampStreamReader();
- }
- return new LongStreamReader();
- } else if (javaType == double.class) {
- return new DoubleStreamReader();
- } else if (javaType == Slice.class) {
- if (type instanceof DecimalType) {
- return new DecimalSliceStreamReader();
- } else {
- if(dictionary != null) {
- return new SliceStreamReader(true, dictionary);
+ if (dictionary != null) {
+ if (javaType == long.class) {
+ if (type instanceof IntegerType || type instanceof DateType) {
+ return new IntegerStreamReader(true, dictionary);
+ } else if (type instanceof DecimalType) {
+ return new DecimalSliceStreamReader(true, dictionary);
+ } else if (type instanceof SmallintType) {
+ return new ShortStreamReader(true, dictionary);
+ }
+ return new LongStreamReader(true, dictionary);
+
+ } else if (javaType == double.class) {
+ return new DoubleStreamReader(true, dictionary);
+ } else if (javaType == Slice.class) {
+ if (type instanceof DecimalType) {
+ return new DecimalSliceStreamReader(true, dictionary);
} else {
- return new SliceStreamReader();
+ return new SliceStreamReader(true, dictionarySliceArrayBlock);
+ }
+ } else {
+ return new ObjectStreamReader();
}
+ } else {
+ if (javaType == long.class) {
+ if (type instanceof IntegerType || type instanceof DateType) {
+ return new IntegerStreamReader();
+ } else if (type instanceof DecimalType) {
+ return new DecimalSliceStreamReader();
+ } else if (type instanceof SmallintType) {
+ return new ShortStreamReader();
+ } else if (type instanceof TimestampType) {
+ return new TimestampStreamReader();
+ }
+ return new LongStreamReader();
+ } else if (javaType == double.class) {
+ return new DoubleStreamReader();
+ } else if (javaType == Slice.class) {
+ if (type instanceof DecimalType) {
+ return new DecimalSliceStreamReader();
+ } else {
+ return new SliceStreamReader();
+ }
+ } else {
+ return new ObjectStreamReader();
}
- } else {
- return new ObjectStreamReader();
+
}
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
index 8e58f9a..9c05177 100644
--- a/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
+++ b/integration/presto/src/main/scala/org/apache/carbondata/presto/CarbonDictionaryDecodeReadSupport.scala
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport
/**
@@ -42,8 +42,7 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
* This initialization is done inside executor task
* for column dictionary involved in decoding.
*
- * @param carbonColumns column list
- * @param absoluteTableIdentifier table identifier
+ * @param carbonColumns column list
*/
override def initialize(carbonColumns: Array[CarbonColumn], carbonTable: CarbonTable) {
@@ -66,9 +65,12 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
dictionaries(index) = forwardDictionaryCache
.get(new DictionaryColumnUniqueIdentifier(carbonTable.getAbsoluteTableIdentifier,
carbonColumn.getColumnIdentifier, dataTypes(index), dictionaryPath))
- dictionarySliceArray(index) = createSliceArrayBlock(dictionaries(index))
-
+ // in case of string data type create dictionarySliceArray same as that of presto code
+ if (dataTypes(index).equals(DataTypes.STRING)) {
+ dictionarySliceArray(index) = createSliceArrayBlock(dictionaries(index))
+ }
}
+
else {
dataTypes(index) = carbonColumn.getDataType
}
@@ -86,17 +88,12 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
val chunks: DictionaryChunksWrapper = dictionaryData.getDictionaryChunks
val sliceArray = new Array[Slice](chunks.getSize + 1)
// Initialize Slice Array with Empty Slice as per Presto's code
- sliceArray(0) = (Slices.EMPTY_SLICE)
+ sliceArray(0) = Slices.EMPTY_SLICE
var count = 1
while (chunks.hasNext) {
{
val value: Array[Byte] = chunks.next
- if (count == 1) {
- sliceArray(count + 1) = null
- }
- else {
- sliceArray(count) = wrappedBuffer(value, 0, value.length)
- }
+ sliceArray(count) = wrappedBuffer(value, 0, value.length)
count += 1
}
}
@@ -104,20 +101,7 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
}
override def readRow(data: Array[AnyRef]): T = {
- throw new RuntimeException("UnSupported Method Call Convert Column Instead")
- }
-
- def convertColumn(data: Array[AnyRef], columnNo: Int): T = {
- val convertedData = if (Option(dictionaries(columnNo)).isDefined) {
- data.map { value =>
- DataTypeUtil
- .getDataBasedOnDataType(dictionaries(columnNo)
- .getDictionaryValueForKey(value.asInstanceOf[Int]), DataTypes.STRING)
- }
- } else {
- data
- }
- convertedData.asInstanceOf[T]
+ throw new RuntimeException("UnSupported Method")
}
/**
@@ -130,6 +114,10 @@ class CarbonDictionaryDecodeReadSupport[T] extends CarbonReadSupport[T] {
dictionarySliceArray(columnNo)
}
+ def getDictionaries: Array[Dictionary] = {
+ dictionaries
+ }
+
/**
* to book keep the dictionary cache or update access count for each
* column involved during decode, to facilitate LRU cache policy if memory
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6add19ad/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index ed89be0..5a0adf5 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -34,28 +34,36 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.hadoop.mapreduce.{RecordReader, TaskType}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier, ReverseDictionary}
+import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier,
+ReverseDictionary}
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl, FileWriteOperation}
-import org.apache.carbondata.core.metadata.converter.{SchemaConverter, ThriftWrapperSchemaConverterImpl}
+import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl,
+FileWriteOperation}
+import org.apache.carbondata.core.metadata.converter.{SchemaConverter,
+ThriftWrapperSchemaConverterImpl}
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, CarbonMeasure, ColumnSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension,
+CarbonMeasure, ColumnSchema}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolutionEntry}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata,
+CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter, CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
-import org.apache.carbondata.core.writer.{CarbonDictionaryWriter, CarbonDictionaryWriterImpl, ThriftWriter}
-import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator, StringArrayWritable}
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter,
+CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
+import org.apache.carbondata.core.writer.{CarbonDictionaryWriter, CarbonDictionaryWriterImpl,
+ThriftWriter}
import org.apache.carbondata.processing.loading.DataLoadExecutor
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
+import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat,
+CSVRecordReaderIterator, StringArrayWritable}
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
+import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.TableOptionConstant
object CarbonDataStoreCreator {
@@ -71,21 +79,20 @@ object CarbonDataStoreCreator {
val dbName: String = "testdb"
val tableName: String = "testtable"
val absoluteTableIdentifier = AbsoluteTableIdentifier.from(
- storePath + "/"+ dbName + "/" + tableName,
+ storePath + "/" + dbName + "/" + tableName,
new CarbonTableIdentifier(dbName,
tableName,
UUID.randomUUID().toString))
- val factFilePath: String = new File(dataFilePath).getCanonicalPath
+ // val factFilePath: String = new File(dataFilePath).getCanonicalPath
val storeDir: File = new File(absoluteTableIdentifier.getTablePath)
CarbonUtil.deleteFoldersAndFiles(storeDir)
CarbonProperties.getInstance.addProperty(
CarbonCommonConstants.STORE_LOCATION_HDFS,
absoluteTableIdentifier.getTablePath)
val table: CarbonTable = createTable(absoluteTableIdentifier)
- writeDictionary(factFilePath, table, absoluteTableIdentifier)
+ writeDictionary(dataFilePath, table, absoluteTableIdentifier)
val schema: CarbonDataLoadSchema = new CarbonDataLoadSchema(table)
val loadModel: CarbonLoadModel = new CarbonLoadModel()
- val partitionId: String = "0"
loadModel.setCarbonDataLoadSchema(schema)
loadModel.setDatabaseName(
absoluteTableIdentifier.getCarbonTableIdentifier.getDatabaseName)
@@ -93,7 +100,7 @@ object CarbonDataStoreCreator {
absoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
loadModel.setTableName(
absoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
- loadModel.setFactFilePath(factFilePath)
+ loadModel.setFactFilePath(dataFilePath)
loadModel.setLoadMetadataDetails(new ArrayList[LoadMetadataDetails]())
loadModel.setTablePath(absoluteTableIdentifier.getTablePath)
CarbonProperties.getInstance
@@ -146,45 +153,52 @@ object CarbonDataStoreCreator {
val tableSchema: TableSchema = new TableSchema()
tableSchema.setTableName(
absoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
- val columnSchemas: List[ColumnSchema] = new ArrayList[ColumnSchema]()
- val encodings: ArrayList[Encoding] = new ArrayList[Encoding]()
- encodings.add(Encoding.INVERTED_INDEX)
+ val columnSchemas = new ArrayList[ColumnSchema]()
+ val dictionaryEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
+ dictionaryEncoding.add(Encoding.DICTIONARY)
+
+ val invertedIndexEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
+ invertedIndexEncoding.add(Encoding.INVERTED_INDEX)
+
val id: ColumnSchema = new ColumnSchema()
id.setColumnName("ID")
id.setColumnar(true)
id.setDataType(DataTypes.INT)
- id.setEncodingList(encodings)
+ id.setEncodingList(dictionaryEncoding)
id.setColumnUniqueId(UUID.randomUUID().toString)
id.setColumnReferenceId(id.getColumnUniqueId)
id.setDimensionColumn(true)
id.setColumnGroup(1)
+ id.setSchemaOrdinal(0)
columnSchemas.add(id)
- val dictEncoding: util.ArrayList[Encoding] = new util.ArrayList[Encoding]()
- dictEncoding.add(Encoding.DIRECT_DICTIONARY)
- dictEncoding.add(Encoding.DICTIONARY)
- dictEncoding.add(Encoding.INVERTED_INDEX)
+ val directDictionaryEncoding: util.ArrayList[Encoding] = new util.ArrayList[Encoding]()
+ directDictionaryEncoding.add(Encoding.DIRECT_DICTIONARY)
+ directDictionaryEncoding.add(Encoding.DICTIONARY)
+ directDictionaryEncoding.add(Encoding.INVERTED_INDEX)
val date: ColumnSchema = new ColumnSchema()
date.setColumnName("date")
date.setColumnar(true)
date.setDataType(DataTypes.DATE)
- date.setEncodingList(dictEncoding)
+ date.setEncodingList(directDictionaryEncoding)
date.setColumnUniqueId(UUID.randomUUID().toString)
date.setDimensionColumn(true)
date.setColumnGroup(2)
date.setColumnReferenceId(date.getColumnUniqueId)
+ date.setSchemaOrdinal(1)
columnSchemas.add(date)
val country: ColumnSchema = new ColumnSchema()
country.setColumnName("country")
country.setColumnar(true)
country.setDataType(DataTypes.STRING)
- country.setEncodingList(encodings)
+ country.setEncodingList(dictionaryEncoding)
country.setColumnUniqueId(UUID.randomUUID().toString)
country.setColumnReferenceId(country.getColumnUniqueId)
country.setDimensionColumn(true)
country.setColumnGroup(3)
+ country.setSchemaOrdinal(2)
country.setColumnReferenceId(country.getColumnUniqueId)
columnSchemas.add(country)
@@ -192,10 +206,11 @@ object CarbonDataStoreCreator {
name.setColumnName("name")
name.setColumnar(true)
name.setDataType(DataTypes.STRING)
- name.setEncodingList(encodings)
+ name.setEncodingList(dictionaryEncoding)
name.setColumnUniqueId(UUID.randomUUID().toString)
name.setDimensionColumn(true)
name.setColumnGroup(4)
+ name.setSchemaOrdinal(3)
name.setColumnReferenceId(name.getColumnUniqueId)
columnSchemas.add(name)
@@ -203,10 +218,11 @@ object CarbonDataStoreCreator {
phonetype.setColumnName("phonetype")
phonetype.setColumnar(true)
phonetype.setDataType(DataTypes.STRING)
- phonetype.setEncodingList(encodings)
+ phonetype.setEncodingList(dictionaryEncoding)
phonetype.setColumnUniqueId(UUID.randomUUID().toString)
phonetype.setDimensionColumn(true)
phonetype.setColumnGroup(5)
+ phonetype.setSchemaOrdinal(4)
phonetype.setColumnReferenceId(phonetype.getColumnUniqueId)
columnSchemas.add(phonetype)
@@ -214,10 +230,11 @@ object CarbonDataStoreCreator {
serialname.setColumnName("serialname")
serialname.setColumnar(true)
serialname.setDataType(DataTypes.STRING)
- serialname.setEncodingList(encodings)
+ serialname.setEncodingList(dictionaryEncoding)
serialname.setColumnUniqueId(UUID.randomUUID().toString)
serialname.setDimensionColumn(true)
serialname.setColumnGroup(6)
+ serialname.setSchemaOrdinal(5)
serialname.setColumnReferenceId(serialname.getColumnUniqueId)
columnSchemas.add(serialname)
@@ -225,10 +242,11 @@ object CarbonDataStoreCreator {
salary.setColumnName("salary")
salary.setColumnar(true)
salary.setDataType(DataTypes.DOUBLE)
- salary.setEncodingList(encodings)
+ salary.setEncodingList(dictionaryEncoding)
salary.setColumnUniqueId(UUID.randomUUID().toString)
- salary.setDimensionColumn(false)
+ salary.setDimensionColumn(true)
salary.setColumnGroup(7)
+ salary.setSchemaOrdinal(6)
salary.setColumnReferenceId(salary.getColumnUniqueId)
columnSchemas.add(salary)
@@ -238,23 +256,25 @@ object CarbonDataStoreCreator {
bonus.setDataType(DataTypes.createDecimalType(10, 4))
bonus.setPrecision(10)
bonus.setScale(4)
- bonus.setEncodingList(encodings)
+ bonus.setEncodingList(invertedIndexEncoding)
bonus.setColumnUniqueId(UUID.randomUUID().toString)
bonus.setDimensionColumn(false)
bonus.setColumnGroup(8)
+ bonus.setSchemaOrdinal(7)
bonus.setColumnReferenceId(bonus.getColumnUniqueId)
columnSchemas.add(bonus)
val monthlyBonus: ColumnSchema = new ColumnSchema()
monthlyBonus.setColumnName("monthlyBonus")
monthlyBonus.setColumnar(true)
- monthlyBonus.setDataType(DataTypes.createDecimalType(18, 4))
- monthlyBonus.setPrecision(18)
+ monthlyBonus.setDataType(DataTypes.createDecimalType(10, 4))
+ monthlyBonus.setPrecision(10)
monthlyBonus.setScale(4)
- monthlyBonus.setEncodingList(encodings)
+ monthlyBonus.setSchemaOrdinal(8)
+ monthlyBonus.setEncodingList(invertedIndexEncoding)
monthlyBonus.setColumnUniqueId(UUID.randomUUID().toString)
monthlyBonus.setDimensionColumn(false)
- monthlyBonus.setColumnGroup(8)
+ monthlyBonus.setColumnGroup(9)
monthlyBonus.setColumnReferenceId(monthlyBonus.getColumnUniqueId)
columnSchemas.add(monthlyBonus)
@@ -262,10 +282,11 @@ object CarbonDataStoreCreator {
dob.setColumnName("dob")
dob.setColumnar(true)
dob.setDataType(DataTypes.TIMESTAMP)
- dob.setEncodingList(dictEncoding)
+ dob.setEncodingList(directDictionaryEncoding)
dob.setColumnUniqueId(UUID.randomUUID().toString)
dob.setDimensionColumn(true)
dob.setColumnGroup(9)
+ dob.setSchemaOrdinal(9)
dob.setColumnReferenceId(dob.getColumnUniqueId)
columnSchemas.add(dob)
@@ -273,10 +294,11 @@ object CarbonDataStoreCreator {
shortField.setColumnName("shortField")
shortField.setColumnar(true)
shortField.setDataType(DataTypes.SHORT)
- shortField.setEncodingList(encodings)
+ shortField.setEncodingList(dictionaryEncoding)
shortField.setColumnUniqueId(UUID.randomUUID().toString)
- shortField.setDimensionColumn(false)
+ shortField.setDimensionColumn(true)
shortField.setColumnGroup(10)
+ shortField.setSchemaOrdinal(10)
shortField.setColumnReferenceId(shortField.getColumnUniqueId)
columnSchemas.add(shortField)
@@ -337,6 +359,7 @@ object CarbonDataStoreCreator {
table.getMeasureByTableName(table.getTableName)
allCols.addAll(msrs)
val set: Array[util.Set[String]] = Array.ofDim[util.Set[String]](dims.size)
+ val dimsIndex = dims.map(dim => dim.getColumnSchema.getSchemaOrdinal)
for (i <- set.indices) {
set(i) = new util.HashSet[String]()
}
@@ -344,15 +367,18 @@ object CarbonDataStoreCreator {
while (line != null) {
val data: Array[String] = line.split(",")
for (i <- set.indices) {
- set(i).add(data(i))
+ set(i).add(data(dimsIndex(i)))
}
line = reader.readLine()
}
val dictCache: Cache[DictionaryColumnUniqueIdentifier, ReverseDictionary] = CacheProvider
.getInstance.createCache(CacheType.REVERSE_DICTIONARY)
+
for (i <- set.indices) {
+ // val dim = getDimension(dims, i).get
val columnIdentifier: ColumnIdentifier =
new ColumnIdentifier(dims.get(i).getColumnId, null, null)
+
val dictionaryColumnUniqueIdentifier: DictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(
table.getAbsoluteTableIdentifier,
@@ -371,7 +397,7 @@ object CarbonDataStoreCreator {
absoluteTableIdentifier,
columnIdentifier,
dims.get(i).getDataType)
- )
+ )
.asInstanceOf[Dictionary]
val preparator: CarbonDictionarySortInfoPreparator =
new CarbonDictionarySortInfoPreparator()
@@ -399,10 +425,11 @@ object CarbonDataStoreCreator {
reader.close()
}
+
/**
* Execute graph which will further load data
*
- * @param loadModel Carbon load model
+ * @param loadModel Carbon load model
* @param storeLocation store location directory
* @throws Exception
*/
@@ -481,25 +508,6 @@ object CarbonDataStoreCreator {
loadModel.getTableName,
loadModel.getTableName,
new ArrayList[LoadMetadataDetails]())
- val segLocation: String = storeLocation + "/" + databaseName + "/" + tableName +
- "/Fact/Part0/Segment_0"
- val file: File = new File(segLocation)
- var factFile: File = null
- val folderList: Array[File] = file.listFiles()
- var folder: File = null
- for (i <- folderList.indices if folderList(i).isDirectory) {
- folder = folderList(i)
- }
- if (folder.isDirectory) {
- val files: Array[File] = folder.listFiles()
- for (i <- files.indices
- if !files(i).isDirectory && files(i).getName.startsWith("part")) {
- factFile = files(i)
- //break
- }
- factFile.renameTo(new File(segLocation + "/" + factFile.getName))
- CarbonUtil.deleteFoldersAndFiles(folder)
- }
}
private def writeLoadMetadata(
[26/50] [abbrv] carbondata git commit: [CARBONDATA-1680] Fixed Bug to
show partition Ids for Hash Partition
Posted by ja...@apache.org.
[CARBONDATA-1680] Fixed Bug to show partition Ids for Hash Partition
This closes #1658
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bc2e897c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bc2e897c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bc2e897c
Branch: refs/heads/fgdatamap
Commit: bc2e897cbe347c9141bb9b83d90ca5aaa42da671
Parents: 5ed39de
Author: SangeetaGulia <sa...@knoldus.in>
Authored: Thu Dec 14 11:34:02 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Dec 21 09:04:37 2017 +0800
----------------------------------------------------------------------
.../spark/testsuite/partition/TestShowPartitions.scala | 5 +++--
.../scala/org/apache/carbondata/spark/util/CommonUtil.scala | 1 +
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc2e897c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
index f455e9a..57739d6 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
@@ -146,7 +146,7 @@ class TestShowPartition extends QueryTest with BeforeAndAfterAll {
test("show partition table: hash table") {
// EqualTo
- checkAnswer(sql("show partitions hashTable"), Seq(Row("empno = HASH_NUMBER(3)")))
+ checkAnswer(sql("show partitions hashTable"), Seq(Row("empno = HASH_NUMBER(3)"), Row("partitionIds = [0, 1, 2]")))
}
@@ -164,7 +164,8 @@ class TestShowPartition extends QueryTest with BeforeAndAfterAll {
}
test("show partition table: not default db") {
// EqualTo
- checkAnswer(sql("show partitions partitionDB.hashTable"), Seq(Row("empno = HASH_NUMBER(3)")))
+ checkAnswer(sql("show partitions partitionDB.hashTable"), Seq(Row("empno = HASH_NUMBER(3)"),
+ Row("partitionIds = [0, 1, 2]")))
// EqualTo
checkAnswer(sql("show partitions partitionDB.rangeTable"), Seq(Row("0, doj = DEFAULT"),
Row("1, doj < 01-01-2010"), Row("2, 01-01-2010 <= doj < 01-01-2015")))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc2e897c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index d8ce8f3..c902537 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -691,6 +691,7 @@ object CommonUtil {
case PartitionType.HASH =>
val hashNumber = partitionInfo.getNumPartitions
result.+=(RowFactory.create(columnName + " = HASH_NUMBER(" + hashNumber.toString() + ")"))
+ result.+=(RowFactory.create(s"partitionIds = ${partitionInfo.getPartitionIds}"))
case others =>
result.+=(RowFactory.create(columnName + " = "))
}
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1480]Min Max Index
Example for DataMap
Posted by ja...@apache.org.
[CARBONDATA-1480]Min Max Index Example for DataMap
Datamap Example. Implementation of Min Max Index through Datamap. And Using the Index while prunning.
This closes #1359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a8114b9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a8114b9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a8114b9b
Branch: refs/heads/fgdatamap
Commit: a8114b9b02de72167361400c508a8c2c41949ddc
Parents: 8a295b5
Author: sounakr <so...@gmail.com>
Authored: Thu Sep 28 16:21:05 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Dec 24 01:19:14 2017 +0800
----------------------------------------------------------------------
.../core/datamap/DataMapStoreManager.java | 16 +-
.../carbondata/core/datamap/TableDataMap.java | 18 +-
.../carbondata/core/datamap/dev/DataMap.java | 14 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../indexstore/SegmentPropertiesFetcher.java | 36 +++
.../blockletindex/BlockletDataMap.java | 9 +-
.../blockletindex/BlockletDataMapFactory.java | 33 ++-
datamap/examples/pom.xml | 111 ++++++++++
.../datamap/examples/BlockletMinMax.java | 41 ++++
.../datamap/examples/MinMaxDataMap.java | 143 ++++++++++++
.../datamap/examples/MinMaxDataMapFactory.java | 114 ++++++++++
.../datamap/examples/MinMaxDataWriter.java | 221 +++++++++++++++++++
.../examples/MinMaxIndexBlockDetails.java | 77 +++++++
.../MinMaxDataMapExample.scala | 77 +++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 2 +-
pom.xml | 2 +
.../datamap/DataMapWriterListener.java | 4 +-
.../store/writer/AbstractFactDataWriter.java | 7 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 3 +
19 files changed, 900 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index d30483a..90e5fff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -103,7 +104,7 @@ public final class DataMapStoreManager {
tableDataMaps = new ArrayList<>();
}
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
- if (dataMap != null) {
+ if (dataMap != null && dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
throw new RuntimeException("Already datamap exists in that path with type " + dataMapName);
}
@@ -113,12 +114,15 @@ public final class DataMapStoreManager {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher);
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher,
+ segmentPropertiesFetcher);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -128,11 +132,11 @@ public final class DataMapStoreManager {
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName,
- List<TableDataMap> tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableDataMaps) {
TableDataMap dataMap = null;
- for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName)) {
+ for (TableDataMap tableDataMap : tableDataMaps) {
+ if (tableDataMap.getDataMapName().equals(dataMapName) || (!tableDataMap.getDataMapName()
+ .equals(""))) {
dataMap = tableDataMap;
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 61d2243..e678920 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -23,9 +23,11 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.events.Event;
@@ -46,15 +48,19 @@ public final class TableDataMap implements OperationEventListener {
private BlockletDetailsFetcher blockletDetailsFetcher;
+ private SegmentPropertiesFetcher segmentPropertiesFetcher;
+
/**
* It is called to initialize and load the required table datamap metadata.
*/
public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
- DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher) {
+ DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
+ SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
this.dataMapName = dataMapName;
this.dataMapFactory = dataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
+ this.segmentPropertiesFetcher = segmentPropertiesFetcher;
}
/**
@@ -67,11 +73,13 @@ public final class TableDataMap implements OperationEventListener {
public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp,
List<String> partitions) throws IOException {
List<ExtendedBlocklet> blocklets = new ArrayList<>();
+ SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp, partitions));
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -123,7 +131,11 @@ public final class TableDataMap implements OperationEventListener {
List<Blocklet> blocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
for (DataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(filterExp, partitions));
+ blocklets.addAll(
+ dataMap.prune(
+ filterExp,
+ segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()),
+ partitions));
}
for (Blocklet blocklet: blocklets) {
ExtendedBlocklet detailedBlocklet =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 16be1ac..dfe97e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -34,23 +35,14 @@ public interface DataMap {
void init(DataMapModel dataMapModel) throws MemoryException, IOException;
/**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- List<Blocklet> prune(FilterResolverIntf filterExp);
-
- // TODO Move this method to Abstract class
- /**
* Prune the datamap with filter expression and partition information. It returns the list of
* blocklets where these filters can exist.
*
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions);
+ List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<String> partitions);
// TODO Move this method to Abstract class
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
index 28163d7..413eaa5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -27,7 +27,7 @@ public interface DataMapWriter {
* Start of new block notification.
* @param blockId file name of the carbondata file
*/
- void onBlockStart(String blockId);
+ void onBlockStart(String blockId, String blockPath);
/**
* End of block notification
@@ -45,7 +45,6 @@ public interface DataMapWriter {
* @param blockletId sequence number of blocklet in the block
*/
void onBlockletEnd(int blockletId);
-
/**
* Add the column pages row to the datamap, order of pages is same as `indexColumns` in
* DataMapMeta returned in DataMapFactory.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
new file mode 100644
index 0000000..ec2ae93
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+
+/**
+ * Fetches the detailed segmentProperties which has more information to execute the query
+ */
+public interface SegmentPropertiesFetcher {
+
+ /**
+ * get the Segment properties based on the SegmentID.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ SegmentProperties getSegmentProperties(String segmentId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index f1188cb..5cb11c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -461,8 +461,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
return false;
}
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp) {
+ private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
// getting the start and end index key based on filter for hitting the
// selected block reference nodes based on filter resolver tree.
@@ -528,7 +527,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
return blocklets;
}
- @Override public List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions) {
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<String> partitions) {
// First get the partitions which are stored inside datamap.
List<String> storedPartitions = getPartitions();
// if it has partitioned datamap but there is no partitioned information stored, it means
@@ -547,7 +548,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
}
// Prune with filters if the partitions are existed in this datamap
- return prune(filterExp);
+ return prune(filterExp, segmentProperties);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 2e2cab5..61e5ceb 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -30,13 +30,18 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -48,13 +53,17 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher {
+public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+ SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+ // segmentId -> SegmentProperties.
+ private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
+
private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
@Override
@@ -170,6 +179,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
+ segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -221,4 +231,25 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// TODO: pass SORT_COLUMNS into this class
return null;
}
+
+ @Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
+ SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
+ if (segmentProperties == null) {
+ int[] columnCardinality;
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ getTableBlockIndexUniqueIdentifiers(segmentId);
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ List<DataFileFooter> indexInfo =
+ fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
+ for (DataFileFooter fileFooter : indexInfo) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ if (segmentProperties == null) {
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ }
+ }
+ segmentPropertiesMap.put(segmentId, segmentProperties);
+ }
+ return segmentProperties;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
new file mode 100644
index 0000000..6832e62
--- /dev/null
+++ b/datamap/examples/pom.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-datamap-examples</artifactId>
+ <name>Apache CarbonData :: Datamap Examples</name>
+
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/minmaxdatamap/main/java</sourceDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>CARBON_EXAMPLESLogResource.properties</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
new file mode 100644
index 0000000..e6968fe
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+
+public class BlockletMinMax {
+ private byte[][] Min;
+
+ private byte[][] Max;
+
+ public byte[][] getMin() {
+ return Min;
+ }
+
+ public void setMin(byte[][] min) {
+ Min = min;
+ }
+
+ public byte[][] getMax() {
+ return Max;
+ }
+
+ public void setMax(byte[][] max) {
+ Max = max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
new file mode 100644
index 0000000..2ad6327
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+/**
+ * Datamap implementation for min max blocklet.
+ */
+public class MinMaxDataMap implements DataMap {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
+
+ public static final String NAME = "clustered.minmax.btree.blocklet";
+
+ private String filePath;
+
+ private MinMaxIndexBlockDetails[] readMinMaxDataMap;
+
+ @Override public void init(String filePath) throws MemoryException, IOException {
+ this.filePath = filePath;
+ CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
+ for (int i = 0; i < listFiles.length; i++) {
+ readMinMaxDataMap = readJson(listFiles[i].getPath());
+ }
+ }
+
+ private CarbonFile[] getCarbonMinMaxIndexFiles(String filePath, String segmentId) {
+ String path = filePath.substring(0, filePath.lastIndexOf("/") + 1);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(path);
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(".minmaxindex");
+ }
+ });
+ }
+
+ public MinMaxIndexBlockDetails[] readJson(String filePath) throws IOException {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ MinMaxIndexBlockDetails[] readMinMax = null;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
+
+ try {
+ if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ buffReader = new BufferedReader(inStream);
+ readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
+ } catch (IOException e) {
+ return null;
+ } finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+ return readMinMax;
+ }
+
+ /**
+ * Block Prunning logic for Min Max DataMap.
+ *
+ * @param filterExp
+ * @param segmentProperties
+ * @return
+ */
+ @Override public List<Blocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties) {
+ List<Blocklet> blocklets = new ArrayList<>();
+
+ if (filterExp == null) {
+ for (int i = 0; i < readMinMaxDataMap.length; i++) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
+ String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ }
+ } else {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ int startIndex = 0;
+ while (startIndex < readMinMaxDataMap.length) {
+ BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
+ readMinMaxDataMap[startIndex].getMinValues());
+ if (!bitSet.isEmpty()) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public void clear() {
+ readMinMaxDataMap = null;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
new file mode 100644
index 0000000..b196d0d
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.events.ChangeEvent;
+import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+
+/**
+ * Min Max DataMap Factory
+ */
+public class MinMaxDataMapFactory implements DataMapFactory {
+
+ private AbsoluteTableIdentifier identifier;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ this.identifier = identifier;
+ }
+
+ /**
+ * createWriter will return the MinMaxDataWriter.
+ * @param segmentId
+ * @return
+ */
+ @Override
+ public DataMapWriter createWriter(String segmentId) {
+ return new MinMaxDataWriter();
+ }
+
+ /**
+ * getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ List<DataMap> dataMapList = new ArrayList<>();
+ // Form a dataMap of Type MinMaxDataMap.
+ MinMaxDataMap dataMap = new MinMaxDataMap();
+ try {
+ dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ } catch (MemoryException ex) {
+
+ }
+ dataMapList.add(dataMap);
+ return dataMapList;
+ }
+
+ /**
+ *
+ * @param segmentId
+ * @return
+ */
+ @Override public List<DataMapDistributable> toDistributable(String segmentId) {
+ return null;
+ }
+
+ /**
+ * Clear the DataMap.
+ * @param segmentId
+ */
+ @Override public void clear(String segmentId) {
+ }
+
+ /**
+ * Clearing the data map.
+ */
+ @Override
+ public void clear() {
+ }
+
+ @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ return null;
+ }
+
+ @Override
+ public void fireEvent(ChangeEvent event) {
+
+ }
+
+ @Override
+ public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
new file mode 100644
index 0000000..78544d3
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class MinMaxDataWriter implements DataMapWriter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(TableInfo.class.getName());
+
+ private byte[][] pageLevelMin, pageLevelMax;
+
+ private byte[][] blockletLevelMin, blockletLevelMax;
+
+ private Map<Integer, BlockletMinMax> blockMinMaxMap;
+
+ private String blockPath;
+
+
+ @Override public void onBlockStart(String blockId, String blockPath) {
+ pageLevelMax = null;
+ pageLevelMin = null;
+ blockletLevelMax = null;
+ blockletLevelMin = null;
+ blockMinMaxMap = null;
+ blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
+ this.blockPath = blockPath;
+ }
+
+ @Override public void onBlockEnd(String blockId) {
+ updateMinMaxIndex(blockId);
+ }
+
+ @Override public void onBlockletStart(int blockletId) {
+ }
+
+ @Override public void onBlockletEnd(int blockletId) {
+ updateBlockletMinMax(blockletId);
+ }
+
+ @Override
+ public void onPageAdded(int blockletId, int pageId, ColumnPage[] pages) {
+ // Calculate Min and Max value within this page.
+
+ // As part of example we are extracting Min Max values Manually. The same can be done from
+ // retrieving the page statistics. For e.g.
+
+ // if (pageLevelMin == null && pageLevelMax == null) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // } else {
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin())) > 0) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // }
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax())) < 0) {
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // }
+
+ byte[] value = new byte[pages[0].getBytes(0).length - 2];
+ if (pageLevelMin == null && pageLevelMax == null) {
+ pageLevelMin = new byte[2][];
+ pageLevelMax = new byte[2][];
+
+ System.arraycopy(pages[0].getBytes(0), 2, value, 0, value.length);
+ pageLevelMin[1] = value;
+ pageLevelMax[1] = value;
+
+ } else {
+ for (int rowIndex = 0; rowIndex < pages[0].getPageSize(); rowIndex++) {
+ System.arraycopy(pages[0].getBytes(rowIndex), 2, value, 0, value.length);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], value) > 0) {
+ pageLevelMin[1] = value;
+ }
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], value) < 0) {
+ pageLevelMax[1] = value;
+ }
+ }
+ }
+ }
+
+ private void updateBlockletMinMax(int blockletId) {
+ if (blockletLevelMax == null || blockletLevelMin == null) {
+ blockletLevelMax = new byte[2][];
+ blockletLevelMin = new byte[2][];
+ if (pageLevelMax != null || pageLevelMin != null) {
+ blockletLevelMin = pageLevelMin;
+ blockletLevelMax = pageLevelMax;
+ }
+ } else {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMin[1], pageLevelMin[1]) > 0) {
+ blockletLevelMin = pageLevelMin;
+ }
+
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMax[1], pageLevelMax[1]) > 0) {
+ blockletLevelMax = pageLevelMax;
+ }
+ }
+ BlockletMinMax blockletMinMax = new BlockletMinMax();
+ blockletMinMax.setMax(blockletLevelMax);
+ blockletMinMax.setMin(blockletLevelMin);
+ blockMinMaxMap.put(blockletId, blockletMinMax);
+ }
+
+
+ public void updateMinMaxIndex(String blockId) {
+ constructMinMaxIndex(blockId);
+ }
+
+
+
+ /**
+ * Construct the Min Max Index.
+ * @param blockId
+ */
+ public void constructMinMaxIndex(String blockId) {
+ // construct Min and Max values of each Blocklets present inside a block.
+ List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
+ tempMinMaxIndexBlockDetails = loadBlockDetails();
+ try {
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ } catch (IOException ex) {
+ LOGGER.info(" Unable to write the file");
+ }
+ }
+
+ /**
+ * loadBlockDetails into the MinMaxIndexBlockDetails class.
+ */
+ private List<MinMaxIndexBlockDetails> loadBlockDetails() {
+ List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails = new ArrayList<MinMaxIndexBlockDetails>();
+ MinMaxIndexBlockDetails tmpminMaxIndexBlockDetails = new MinMaxIndexBlockDetails();
+
+ for (int index = 0; index < blockMinMaxMap.size(); index++) {
+ tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
+ tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
+ tmpminMaxIndexBlockDetails.setBlockletId(index);
+ tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
+ minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
+ }
+ return minMaxIndexBlockDetails;
+ }
+
+ /**
+ * Write the data to a file. This is JSON format file.
+ * @param minMaxIndexBlockDetails
+ * @param blockPath
+ * @param blockId
+ * @throws IOException
+ */
+ public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
+ String blockPath, String blockId) throws IOException {
+ String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
+ + ".minmaxindex";
+ BufferedWriter brWriter = null;
+ DataOutputStream dataOutStream = null;
+ try {
+ FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
+ dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
+ Gson gsonObjectToWrite = new Gson();
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
+ brWriter.write(minmaxIndexData);
+ } catch (IOException ioe) {
+ LOGGER.info("Error in writing minMaxindex file");
+ } finally {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ if (null != dataOutStream) {
+ dataOutStream.flush();
+ }
+ CarbonUtil.closeStreams(brWriter, dataOutStream);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
new file mode 100644
index 0000000..0596db5
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.Serializable;
+
+public class MinMaxIndexBlockDetails implements Serializable {
+ private static final long serialVersionUID = 1206104914911491724L;
+
+ /**
+ * Min value of a column of one blocklet Bit-Packed
+ */
+ private byte[][] minValues;
+
+ /**
+ * Max value of a columns of one blocklet Bit-Packed
+ */
+ private byte[][] maxValues;
+
+ /**
+ * filePath pointing to the block.
+ */
+ private String filePath;
+
+ /**
+ * BlockletID of the block.
+ */
+ private Integer BlockletId;
+
+
+ public byte[][] getMinValues() {
+ return minValues;
+ }
+
+ public void setMinValues(byte[][] minValues) {
+ this.minValues = minValues;
+ }
+
+ public byte[][] getMaxValues() {
+ return maxValues;
+ }
+
+ public void setMaxValues(byte[][] maxValues) {
+ this.maxValues = maxValues;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public Integer getBlockletId() {
+ return BlockletId;
+ }
+
+ public void setBlockletId(Integer blockletId) {
+ BlockletId = blockletId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
new file mode 100644
index 0000000..0cfe410
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.datamap.examples
+
+import java.io.File
+
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.CarbonProperties
+
+object MinMaxDataMapExample {
+ def main(args: Array[String]): Unit = {
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "").getCanonicalPath
+ val storeLocation = s"$rootPath/dataMap/examples/target/store"
+ val warehouse = s"$rootPath/datamap/examples/target/warehouse"
+ val metastoredb = s"$rootPath/datamap/examples/target"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonDataMapExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation)
+
+ spark.sparkContext.setLogLevel("ERROR")
+ import spark.implicits._
+
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ AbsoluteTableIdentifier.from(storeLocation, "default", "carbonminmax"),
+ classOf[MinMaxDataMapFactory].getName,
+ MinMaxDataMap.NAME)
+
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ val df = spark.sparkContext.parallelize(1 to 33000)
+ .map(x => ("a", "b", x))
+ .toDF("c1", "c2", "c3")
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbonminmax")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ // Query the table.
+ spark.sql("select c2 from carbonminmax").show(20, false)
+ spark.sql("select c2 from carbonminmax where c2 = 'b'").show(20, false)
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 04a5f9c..6f4e9cd 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -193,7 +193,7 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String): Unit = {
+ override def onBlockStart(blockId: String, blockPath: String): Unit = {
callbackSeq :+= s"block start $blockId"
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index be49b75..a2f11a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -435,6 +435,7 @@
<module>format</module>
<module>integration/spark2</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>examples/flink</module>
@@ -474,6 +475,7 @@
<module>integration/presto</module>
<module>streaming</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
</modules>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 4b0113c..8e350d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -82,10 +82,10 @@ public class DataMapWriterListener {
LOG.info("DataMapWriter " + writer + " added");
}
- public void onBlockStart(String blockId) {
+ public void onBlockStart(String blockId, String blockPath) {
for (List<DataMapWriter> writers : registry.values()) {
for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId);
+ writer.onBlockStart(blockId, blockPath);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 855ec03..b00ec70 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -268,7 +268,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
private void notifyDataMapBlockStart() {
if (listener != null) {
- listener.onBlockStart(carbonDataFileName);
+ listener.onBlockStart(carbonDataFileName, constructFactFileFullPath());
}
}
@@ -279,6 +279,11 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
blockletId = 0;
}
+ private String constructFactFileFullPath() {
+ String factFilePath =
+ this.dataWriterVo.getCarbonDataDirectoryPath() + File.separator + this.carbonDataFileName;
+ return factFilePath;
+ }
/**
* Finish writing current file. It will flush stream, copy and rename temp file to final file
* @param copyInCurrentThread set to false if want to do data copy in a new thread
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8114b9b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index d8ae8ff..c366f63 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.processing.store.TablePage;
import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
+
/**
* Below class will be used to write the data in V3 format
* <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
@@ -157,6 +158,8 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
}
}
+
+
/**
* Write the collect blocklet data (blockletDataHolder) to file
*/
[17/50] [abbrv] carbondata git commit: [CARBONDATA-1854] add support
for implicit column filter
Posted by ja...@apache.org.
[CARBONDATA-1854] add support for implicit column filter
This closes #1644
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/28c94183
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/28c94183
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/28c94183
Branch: refs/heads/fgdatamap
Commit: 28c94183bf5e03f6af05a270b96c30529233332d
Parents: b8a02f3
Author: rahulforallp <ra...@knoldus.in>
Authored: Tue Dec 12 14:49:27 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Dec 20 08:18:52 2017 +0530
----------------------------------------------------------------------
.../scan/filter/FilterExpressionProcessor.java | 11 ++++
.../filter/FilterExpressionProcessorTest.java | 68 ++++++++++++++++++++
.../load/DataLoadProcessorStepOnSpark.scala | 5 ++
.../spark/rdd/CarbonDataRDDFactory.scala | 5 ++
4 files changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/28c94183/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 6c804d7..5a1b7df 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -403,6 +403,17 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
case EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
+ // check for implicit column in the expression
+ if (currentCondExpression instanceof InExpression) {
+ CarbonColumn carbonColumn =
+ currentCondExpression.getColumnList().get(0).getCarbonColumn();
+ if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) {
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier,
+ currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
+ }
+ }
+
CarbonColumn column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
if (currentCondExpression.isSingleColumn() && ! column.getDataType().isComplexType()) {
if (column.isMeasure()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/28c94183/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
new file mode 100644
index 0000000..1f244a3
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
@@ -0,0 +1,68 @@
+package org.apache.carbondata.core.scan.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.InExpression;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class FilterExpressionProcessorTest extends AbstractDictionaryCacheTest {
+
+ private ColumnSchema columnSchema;
+
+ @Before public void setUp() throws Exception {
+ init();
+ this.databaseName = props.getProperty("database", "testSchema");
+ this.tableName = props.getProperty("tableName", "carbon");
+ this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+ carbonTableIdentifier =
+ new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
+ this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+ columnSchema = new ColumnSchema();
+ columnSchema.setColumnar(true);
+ columnSchema.setColumnName("IMEI");
+ columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
+ columnSchema.setDataType(DataTypes.STRING);
+ columnSchema.setDimensionColumn(true);
+ List<Encoding> encodingList = new ArrayList<>();
+ encodingList.add(Encoding.IMPLICIT);
+ columnSchema.setEncodingList(encodingList);
+ }
+
+ @Test public void testGetFilterResolverBasedOnExpressionType()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ CarbonColumn carbonColumn = new CarbonColumn(columnSchema, 0, 0);
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
+ columnExpression.setCarbonColumn(carbonColumn);
+ LiteralExpression literalExpression = new LiteralExpression("ImeiValue", DataTypes.STRING);
+ InExpression equalToExpression = new InExpression(columnExpression, literalExpression);
+ FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
+ Method method = FilterExpressionProcessor.class
+ .getDeclaredMethod("getFilterResolverBasedOnExpressionType", ExpressionType.class,
+ boolean.class, Expression.class, AbsoluteTableIdentifier.class, Expression.class);
+ method.setAccessible(true);
+ Object result = method
+ .invoke(filterExpressionProcessor, ExpressionType.EQUALS, false, equalToExpression, null,
+ null);
+ assert (result.getClass().getName()
+ .equals("org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/28c94183/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 6759b20..5e6ba98 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -19,6 +19,7 @@ package org.apache.carbondata.spark.load
import scala.util.Random
+import com.univocity.parsers.common.TextParsingException
import org.apache.spark.{Accumulator, SparkEnv, TaskContext}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.Row
@@ -234,6 +235,10 @@ object DataLoadProcessorStepOnSpark {
private def wrapException(e: Throwable, model: CarbonLoadModel): Unit = {
e match {
case e: CarbonDataLoadingException => throw e
+ case e: TextParsingException =>
+ LOGGER.error(e, "Data Loading failed for table " + model.getTableName)
+ throw new CarbonDataLoadingException("Data Loading failed for table " + model.getTableName,
+ e)
case e: Exception =>
LOGGER.error(e, "Data Loading failed for table " + model.getTableName)
throw new CarbonDataLoadingException("Data Loading failed for table " + model.getTableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/28c94183/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 29712de..0b786b5 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -27,6 +27,7 @@ import scala.collection.mutable.ListBuffer
import scala.util.Random
import scala.util.control.Breaks._
+import com.univocity.parsers.common.TextParsingException
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.NullWritable
@@ -403,6 +404,10 @@ object CarbonDataRDDFactory {
sparkException.getCause.isInstanceOf[CarbonDataLoadingException]) {
executorMessage = sparkException.getCause.getMessage
errorMessage = errorMessage + ": " + executorMessage
+ } else if (sparkException.getCause.isInstanceOf[TextParsingException]) {
+ executorMessage = CarbonDataProcessorUtil
+ .trimErrorMessage(sparkException.getCause.getMessage)
+ errorMessage = errorMessage + " : " + executorMessage
}
case _ =>
if (ex.getCause != null) {
[33/50] [abbrv] carbondata git commit: [CARBONDATA-1924][PARTITION]
Restrict streaming on Partitioned table and support PARTITION syntax to the
LOAD TABLE command.
Posted by ja...@apache.org.
[CARBONDATA-1924][PARTITION] Restrict streaming on Partitioned table and support PARTITION syntax to the LOAD TABLE command.
This closes #1699
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/577a8b0d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/577a8b0d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/577a8b0d
Branch: refs/heads/fgdatamap
Commit: 577a8b0d59d760a35626779fe45dd4d91c6b3328
Parents: a89587e
Author: ravipesala <ra...@gmail.com>
Authored: Tue Dec 19 20:01:30 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Dec 21 18:46:50 2017 +0530
----------------------------------------------------------------------
.../StandardPartitionTableLoadingTestCase.scala | 57 ++++++++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 6 +++
.../management/CarbonLoadDataCommand.scala | 10 +++-
.../sql/parser/CarbonSpark2SqlParser.scala | 22 +++++---
.../spark/sql/parser/CarbonSparkSqlParser.scala | 3 ++
5 files changed, 89 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/577a8b0d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 11e95d8..b7010e5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -17,6 +17,7 @@
package org.apache.carbondata.spark.testsuite.standardpartition
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{AnalysisException, Row}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -200,6 +201,59 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
validateDataFiles("default_singlepasspartitionone", "0", 1)
}
+ test("data loading for partition table for one static partition column with load syntax") {
+ sql(
+ """
+ | CREATE TABLE loadstaticpartitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitionone PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select distinct empno from loadstaticpartitionone"), Seq(Row(1)))
+ }
+
+ test("overwrite partition table for one static partition column with load syntax") {
+ sql(
+ """
+ | CREATE TABLE loadstaticpartitiononeoverwrite (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql("select count(*) from loadstaticpartitiononeoverwrite").collect()
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select count(*) from loadstaticpartitiononeoverwrite"), rows)
+ }
+
+ test("Restrict streaming on partitioned table") {
+ intercept[AnalysisException] {
+ sql(
+ """
+ | CREATE TABLE streamingpartitionedtable (empname String, designation String, doj
+ | Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('streaming'='true')
+ """.stripMargin)
+ }
+ }
+
override def afterAll = {
dropTable
@@ -215,6 +269,9 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
sql("drop table if exists insertpartitionthree")
sql("drop table if exists staticpartitionone")
sql("drop table if exists singlepasspartitionone")
+ sql("drop table if exists loadstaticpartitionone")
+ sql("drop table if exists loadstaticpartitiononeoverwrite")
+ sql("drop table if exists streamingpartitionedtable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/577a8b0d/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 6e9b36c..129e6b3 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -978,6 +978,12 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
case _ => ("", "")
}
+ protected lazy val partitions: Parser[(String, String)] =
+ (ident <~ "=") ~ stringLit ^^ {
+ case opt ~ optvalue => (opt.trim, optvalue)
+ case _ => ("", "")
+ }
+
protected lazy val valueOptions: Parser[(Int, Int)] =
(numericLit <~ ",") ~ numericLit ^^ {
case opt ~ optvalue => (opt.toInt, optvalue.toInt)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/577a8b0d/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 7492951..f96c0a7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -558,8 +558,14 @@ case class CarbonLoadDataCommand(
}
// Only select the required columns
- Project(relation.output.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get),
- LogicalRDD(attributes, rdd)(sparkSession))
+ val output = if (partition.nonEmpty) {
+ relation.output.map{ attr =>
+ attributes.find(_.name.equalsIgnoreCase(attr.name)).get
+ }.filter(attr => partition.get(attr.name).isEmpty)
+ } else {
+ relation.output.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
+ }
+ Project(output, LogicalRDD(attributes, rdd)(sparkSession))
}
val convertRelation = relation match {
case l: LogicalRelation =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/577a8b0d/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index dad0e3e..5d00a0c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -351,8 +351,9 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
protected lazy val loadDataNew: Parser[LogicalPlan] =
LOAD ~> DATA ~> opt(LOCAL) ~> INPATH ~> stringLit ~ opt(OVERWRITE) ~
(INTO ~> TABLE ~> (ident <~ ".").? ~ ident) ~
+ (PARTITION ~>"("~> repsep(partitions, ",") <~ ")").? ~
(OPTIONS ~> "(" ~> repsep(loadOptions, ",") <~ ")").? <~ opt(";") ^^ {
- case filePath ~ isOverwrite ~ table ~ optionsList =>
+ case filePath ~ isOverwrite ~ table ~ partitions ~ optionsList =>
val (databaseNameOp, tableName) = table match {
case databaseName ~ tableName => (databaseName, tableName.toLowerCase())
}
@@ -360,13 +361,20 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
validateOptions(optionsList)
}
val optionsMap = optionsList.getOrElse(List.empty[(String, String)]).toMap
+ val partitionSpec = partitions.getOrElse(List.empty[(String, String)]).toMap
CarbonLoadDataCommand(
- convertDbNameToLowerCase(databaseNameOp),
- tableName,
- filePath,
- Seq(),
- optionsMap,
- isOverwrite.isDefined)
+ databaseNameOp = convertDbNameToLowerCase(databaseNameOp),
+ tableName = tableName,
+ factPathFromUser = filePath,
+ dimFilesPath = Seq(),
+ options = optionsMap,
+ isOverwriteTable = isOverwrite.isDefined,
+ inputSqlString = null,
+ dataFrame = None,
+ updateModel = None,
+ tableInfoOp = None,
+ internalOptions = Map.empty,
+ partition = partitionSpec.map { case (key, value) => (key, Some(value))})
}
protected lazy val deleteLoadsByID: Parser[LogicalPlan] =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/577a8b0d/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 3597208..211e0ef 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -233,6 +233,9 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
case _ =>
// ignore this case
}
+ if (partitionFields.nonEmpty && options.isStreaming) {
+ operationNotAllowed("Streaming is not allowed on partitioned table", partitionColumns)
+ }
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
// prepare table model of the collected tokens
[50/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap]
Datamap FineGrain implementation
Posted by ja...@apache.org.
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation
Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap.
FG Query flow as follows.
1.The user can add FG datamap to any table and implement there interfaces.
2. Any filter query which hits the table with datamap will call prune method of FGdatamap.
3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well.
4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits.
5. Based on the splits scanrdd schedule the tasks.
6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it.
This closes #1471
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/65d05375
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/65d05375
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/65d05375
Branch: refs/heads/fgdatamap
Commit: 65d05375f18f14f832ba85b9508a8594059d9bb0
Parents: a8114b9
Author: ravipesala <ra...@gmail.com>
Authored: Wed Nov 15 19:48:40 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Dec 24 01:36:11 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapMeta.java | 8 +-
.../core/datamap/DataMapStoreManager.java | 30 +-
.../carbondata/core/datamap/DataMapType.java | 21 +
.../carbondata/core/datamap/TableDataMap.java | 31 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 110 +++++
.../core/datamap/dev/BlockletSerializer.java | 57 +++
.../carbondata/core/datamap/dev/DataMap.java | 4 +-
.../core/datamap/dev/DataMapFactory.java | 14 +-
.../core/datamap/dev/DataMapWriter.java | 57 ---
.../cgdatamap/AbstractCoarseGrainDataMap.java | 24 +
.../AbstractCoarseGrainDataMapFactory.java | 34 ++
.../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 +
.../AbstractFineGrainDataMapFactory.java | 38 ++
.../carbondata/core/datastore/DataRefNode.java | 7 +
.../core/datastore/block/TableBlockInfo.java | 10 +
.../impl/btree/AbstractBTreeLeafNode.java | 5 +
.../datastore/impl/btree/BTreeNonLeafNode.java | 5 +
.../carbondata/core/indexstore/Blocklet.java | 30 +-
.../core/indexstore/BlockletDetailsFetcher.java | 8 +
.../core/indexstore/ExtendedBlocklet.java | 19 +-
.../core/indexstore/FineGrainBlocklet.java | 120 +++++
.../blockletindex/BlockletDataMap.java | 9 +-
.../blockletindex/BlockletDataMapFactory.java | 62 ++-
.../BlockletDataRefNodeWrapper.java | 30 +-
.../indexstore/blockletindex/IndexWrapper.java | 18 +
.../core/indexstore/schema/FilterType.java | 24 -
.../executer/ExcludeFilterExecuterImpl.java | 3 +
.../executer/IncludeFilterExecuterImpl.java | 3 +
.../core/scan/processor/BlocksChunkHolder.java | 5 -
.../core/scan/scanner/impl/FilterScanner.java | 2 +
.../apache/carbondata/core/util/CarbonUtil.java | 97 +++++
.../datamap/examples/MinMaxDataMap.java | 20 +-
.../datamap/examples/MinMaxDataMapFactory.java | 49 ++-
.../datamap/examples/MinMaxDataWriter.java | 36 +-
.../examples/MinMaxIndexBlockDetails.java | 13 -
.../carbondata/hadoop/CarbonInputFormat.java | 2 +-
.../carbondata/hadoop/CarbonInputSplit.java | 20 +-
.../hadoop/api/CarbonTableInputFormat.java | 20 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 357 +++++++++++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 46 +-
.../testsuite/datamap/FGDataMapTestCase.scala | 436 +++++++++++++++++++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 7 +-
.../datamap/DataMapWriterListener.java | 57 ++-
.../store/CarbonFactDataHandlerModel.java | 10 +-
.../store/writer/AbstractFactDataWriter.java | 126 +-----
45 files changed, 1730 insertions(+), 378 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
index 7746acf..dd15ccb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
@@ -19,15 +19,15 @@ package org.apache.carbondata.core.datamap;
import java.util.List;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
public class DataMapMeta {
private List<String> indexedColumns;
- private FilterType optimizedOperation;
+ private List<ExpressionType> optimizedOperation;
- public DataMapMeta(List<String> indexedColumns, FilterType optimizedOperation) {
+ public DataMapMeta(List<String> indexedColumns, List<ExpressionType> optimizedOperation) {
this.indexedColumns = indexedColumns;
this.optimizedOperation = optimizedOperation;
}
@@ -36,7 +36,7 @@ public class DataMapMeta {
return indexedColumns;
}
- public FilterType getOptimizedOperation() {
+ public List<ExpressionType> getOptimizedOperation() {
return optimizedOperation;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 90e5fff..8d80b4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -56,7 +56,22 @@ public final class DataMapStoreManager {
}
public List<TableDataMap> getAllDataMap(AbsoluteTableIdentifier identifier) {
- return allDataMaps.get(identifier.uniqueName());
+ return allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
+ }
+
+ // TODO its a temporary method till chooser is implemented
+ public TableDataMap chooseDataMap(AbsoluteTableIdentifier identifier) {
+ List<TableDataMap> tableDataMaps = getAllDataMap(identifier);
+ if (tableDataMaps != null && tableDataMaps.size() > 0) {
+ for (TableDataMap dataMap: tableDataMaps) {
+ if (!dataMap.getDataMapName().equalsIgnoreCase(BlockletDataMap.NAME)) {
+ return dataMap;
+ }
+ }
+ return tableDataMaps.get(0);
+ } else {
+ return getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
+ }
}
/**
@@ -68,7 +83,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap getDataMap(AbsoluteTableIdentifier identifier,
String dataMapName, String factoryClass) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
TableDataMap dataMap;
if (tableDataMaps == null) {
@@ -96,7 +111,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
String factoryClassName, String dataMapName) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
@@ -149,7 +164,9 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(tableUniqueName);
segmentRefreshMap.remove(identifier.uniqueName());
if (tableDataMaps != null) {
for (TableDataMap tableDataMap: tableDataMaps) {
@@ -158,7 +175,7 @@ public final class DataMapStoreManager {
break;
}
}
- allDataMaps.remove(identifier.uniqueName());
+ allDataMaps.remove(tableUniqueName);
}
}
@@ -167,7 +184,8 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
if (tableDataMaps != null) {
int i = 0;
for (TableDataMap tableDataMap: tableDataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
new file mode 100644
index 0000000..bf812b3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+public enum DataMapType {
+ CG,FG;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index e678920..bd8fd98 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -21,12 +21,16 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -76,10 +80,15 @@ public final class TableDataMap implements OperationEventListener {
SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
- segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
- for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
+ // if filter is not passed then return all the blocklets
+ if (filterExp == null) {
+ pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segmentId);
+ } else {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
+ for (DataMap dataMap : dataMaps) {
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
+ }
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -137,9 +146,21 @@ public final class TableDataMap implements OperationEventListener {
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()),
partitions));
}
- for (Blocklet blocklet: blocklets) {
+ BlockletSerializer serializer = new BlockletSerializer();
+ String writePath =
+ identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapName;
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
+ }
+ for (Blocklet blocklet : blocklets) {
ExtendedBlocklet detailedBlocklet =
blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegmentId());
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ String blockletwritePath =
+ writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
+ detailedBlocklet.setDataMapWriterPath(blockletwritePath);
+ serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
+ }
detailedBlocklet.setSegmentId(distributable.getSegmentId());
detailedBlocklets.add(detailedBlocklet);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
new file mode 100644
index 0000000..bcc9bad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Data Map writer
+ */
+public abstract class AbstractDataMapWriter {
+
+ protected AbsoluteTableIdentifier identifier;
+
+ protected String segmentId;
+
+ protected String writeDirectoryPath;
+
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String writeDirectoryPath) {
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.writeDirectoryPath = writeDirectoryPath;
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ public abstract void onBlockStart(String blockId);
+
+ /**
+ * End of block notification
+ */
+ public abstract void onBlockEnd(String blockId);
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletStart(int blockletId);
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletEnd(int blockletId);
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ public abstract void finish() throws IOException;
+
+ /**
+ * It copies the file from temp folder to actual folder
+ *
+ * @param dataMapFile
+ * @throws IOException
+ */
+ protected void commitFile(String dataMapFile) throws IOException {
+ if (!dataMapFile.startsWith(writeDirectoryPath)) {
+ throw new UnsupportedOperationException(
+ "Datamap file " + dataMapFile + " is not written in provided directory path "
+ + writeDirectoryPath);
+ }
+ String dataMapFileName =
+ dataMapFile.substring(writeDirectoryPath.length(), dataMapFile.length());
+ String carbonFilePath = dataMapFileName.substring(0, dataMapFileName.lastIndexOf("/"));
+ String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
+ if (carbonFilePath.length() > 0) {
+ carbonFilePath = segmentPath + carbonFilePath;
+ FileFactory.mkdirs(carbonFilePath, FileFactory.getFileType(carbonFilePath));
+ } else {
+ carbonFilePath = segmentPath;
+ }
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(dataMapFile, carbonFilePath, 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
new file mode 100644
index 0000000..3d4c717
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public class BlockletSerializer {
+
+ /**
+ * Serialize and write blocklet to the file.
+ * @param grainBlocklet
+ * @param writePath
+ * @throws IOException
+ */
+ public void serializeBlocklet(FineGrainBlocklet grainBlocklet, String writePath)
+ throws IOException {
+ DataOutputStream dataOutputStream =
+ FileFactory.getDataOutputStream(writePath, FileFactory.getFileType(writePath));
+ grainBlocklet.write(dataOutputStream);
+ dataOutputStream.close();
+ }
+
+ /**
+ * Read data from filepath and deserialize blocklet.
+ * @param writePath
+ * @return
+ * @throws IOException
+ */
+ public FineGrainBlocklet deserializeBlocklet(String writePath) throws IOException {
+ DataInputStream inputStream =
+ FileFactory.getDataInputStream(writePath, FileFactory.getFileType(writePath));
+ FineGrainBlocklet blocklet = new FineGrainBlocklet();
+ blocklet.readFields(inputStream);
+ inputStream.close();
+ return blocklet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index dfe97e3..3fa6d75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
* Datamap is an entity which can store and retrieve index data.
*/
-public interface DataMap {
+public interface DataMap<T extends Blocklet> {
/**
* It is called to load the data map to memory or to initialize it.
@@ -41,7 +41,7 @@ public interface DataMap {
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
List<String> partitions);
// TODO Move this method to Abstract class
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index f5a7404..e900f8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,13 +21,14 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.events.Event;
/**
* Interface for datamap factory, it is responsible for creating the datamap.
*/
-public interface DataMapFactory {
+public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
@@ -37,17 +38,17 @@ public interface DataMapFactory {
/**
* Return a new write for this datamap
*/
- DataMapWriter createWriter(String segmentId);
+ AbstractDataMapWriter createWriter(String segmentId, String writeDirectoryPath);
/**
* Get the datamap for segmentid
*/
- List<DataMap> getDataMaps(String segmentId) throws IOException;
+ List<T> getDataMaps(String segmentId) throws IOException;
/**
* Get datamaps for distributable object.
*/
- List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException;
+ List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
/**
* Get all distributable objects of a segmentid
@@ -75,4 +76,9 @@ public interface DataMapFactory {
* Return metadata of this datamap
*/
DataMapMeta getMeta();
+
+ /**
+ * Type of datamap whether it is FG or CG
+ */
+ DataMapType getDataMapType();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
deleted file mode 100644
index 413eaa5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-
-/**
- * Data Map writer
- */
-public interface DataMapWriter {
-
- /**
- * Start of new block notification.
- * @param blockId file name of the carbondata file
- */
- void onBlockStart(String blockId, String blockPath);
-
- /**
- * End of block notification
- */
- void onBlockEnd(String blockId);
-
- /**
- * Start of new blocklet notification.
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletStart(int blockletId);
-
- /**
- * End of blocklet notification
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletEnd(int blockletId);
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
new file mode 100644
index 0000000..d79d0c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.Blocklet;
+
+public abstract class AbstractCoarseGrainDataMap implements DataMap<Blocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
new file mode 100644
index 0000000..9789992
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
+ * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
+ * information of block and blocklet.
+ * 3. Based on the splits scanrdd schedule the tasks.
+ */
+public abstract class AbstractCoarseGrainDataMapFactory
+ implements DataMapFactory<AbstractCoarseGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.CG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
new file mode 100644
index 0000000..310fb3b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public abstract class AbstractFineGrainDataMap implements DataMap<FineGrainBlocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
new file mode 100644
index 0000000..1ca7fc3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
+ * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
+ * information of block, blocklet, page and rowids information as well.
+ * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
+ * blocklet and filepath information as part of Splits.
+ * 4. Based on the splits scanrdd schedule the tasks.
+ * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
+ * bitset if exists. And pass this bitset as input to it.
+ */
+public abstract class AbstractFineGrainDataMapFactory
+ implements DataMapFactory<AbstractFineGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.FG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 37acb02..f258a88 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Interface data block reference
@@ -124,6 +125,12 @@ public interface DataRefNode {
BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache();
/**
+ * Return the indexed data if it has any from disk which was stored by FG datamap.
+ * @return
+ */
+ BitSetGroup getIndexedData();
+
+ /**
* number of pages in blocklet
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 910c9bb..a1ef45f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -85,6 +85,8 @@ public class TableBlockInfo implements Distributable, Serializable {
private BlockletDetailInfo detailInfo;
+ private String dataMapWriterPath;
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId, String[] locations,
long blockLength, ColumnarFormatVersion version, String[] deletedDeltaFilePath) {
this.filePath = FileFactory.getUpdatedFilePath(filePath);
@@ -356,4 +358,12 @@ public class TableBlockInfo implements Distributable, Serializable {
public void setDetailInfo(BlockletDetailInfo detailInfo) {
this.detailInfo = detailInfo;
}
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 19b1f1c..952b7b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Non leaf node abstract class
@@ -244,4 +245,8 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
public int getPageRowCount(int pageNumber) {
throw new UnsupportedOperationException("Unsupported operation");
}
+
+ @Override public BitSetGroup getIndexedData() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index 62fcabf..ec5eca6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* No leaf node of a b+tree class which will keep the matadata(start key) of the
@@ -245,6 +246,10 @@ public class BTreeNonLeafNode implements BTreeNode {
return deleteDeltaDataCache;
}
+ public BitSetGroup getIndexedData() {
+ return null;
+ }
+
/**
* number of pages in blocklet
* @return
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index d84f3f6..c731e07 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -16,28 +16,46 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+
/**
* Blocklet
*/
-public class Blocklet implements Serializable {
+public class Blocklet implements Writable,Serializable {
- private String path;
+ private String blockId;
private String blockletId;
- public Blocklet(String path, String blockletId) {
- this.path = path;
+ public Blocklet(String blockId, String blockletId) {
+ this.blockId = blockId;
this.blockletId = blockletId;
}
- public String getPath() {
- return path;
+ // For serialization purpose
+ public Blocklet() {
}
public String getBlockletId() {
return blockletId;
}
+ public String getBlockId() {
+ return blockId;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeUTF(blockId);
+ out.writeUTF(blockletId);
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ blockId = in.readUTF();
+ blockletId = in.readUTF();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
index 21ecba1..a493c06 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
@@ -44,4 +44,12 @@ public interface BlockletDetailsFetcher {
* @throws IOException
*/
ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, String segmentId) throws IOException;
+
+ /**
+ * Get all the blocklets in a segment
+ *
+ * @param segmentId
+ * @return
+ */
+ List<Blocklet> getAllBlocklets(String segmentId) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
index e0cfefb..081da53 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
@@ -38,8 +38,13 @@ public class ExtendedBlocklet extends Blocklet {
private String[] location;
+ private String path;
+
+ private String dataMapWriterPath;
+
public ExtendedBlocklet(String path, String blockletId) {
super(path, blockletId);
+ this.path = path;
}
public BlockletDetailInfo getDetailInfo() {
@@ -56,7 +61,7 @@ public class ExtendedBlocklet extends Blocklet {
* @throws IOException
*/
public void updateLocations() throws IOException {
- Path path = new Path(getPath());
+ Path path = new Path(this.path);
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
LocatedFileStatus fileStatus = iter.next();
@@ -79,4 +84,16 @@ public class ExtendedBlocklet extends Blocklet {
public void setSegmentId(String segmentId) {
this.segmentId = segmentId;
}
+
+ public String getPath() {
+ return path;
+ }
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
new file mode 100644
index 0000000..266120e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * FineGrainBlocklet
+ */
+public class FineGrainBlocklet extends Blocklet implements Serializable {
+
+ private List<Page> pages;
+
+ public FineGrainBlocklet(String blockId, String blockletId, List<Page> pages) {
+ super(blockId, blockletId);
+ this.pages = pages;
+ }
+
+ // For serialization purpose
+ public FineGrainBlocklet() {
+
+ }
+
+ public List<Page> getPages() {
+ return pages;
+ }
+
+ public static class Page implements Writable,Serializable {
+
+ private int pageId;
+
+ private int[] rowId;
+
+ public BitSet getBitSet() {
+ BitSet bitSet =
+ new BitSet(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ for (int row : rowId) {
+ bitSet.set(row);
+ }
+ return bitSet;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(pageId);
+ out.writeInt(rowId.length);
+ for (int i = 0; i < rowId.length; i++) {
+ out.writeInt(rowId[i]);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ pageId = in.readInt();
+ int length = in.readInt();
+ rowId = new int[length];
+ for (int i = 0; i < length; i++) {
+ rowId[i] = in.readInt();
+ }
+ }
+
+ public void setPageId(int pageId) {
+ this.pageId = pageId;
+ }
+
+ public void setRowId(int[] rowId) {
+ this.rowId = rowId;
+ }
+ }
+
+ public BitSetGroup getBitSetGroup(int numberOfPages) {
+ BitSetGroup bitSetGroup = new BitSetGroup(numberOfPages);
+ for (int i = 0; i < pages.size(); i++) {
+ bitSetGroup.setBitSet(pages.get(i).getBitSet(), pages.get(i).pageId);
+ }
+ return bitSetGroup;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ super.write(out);
+ int size = pages.size();
+ out.writeInt(size);
+ for (Page page : pages) {
+ page.write(out);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ pages = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ Page page = new Page();
+ page.readFields(in);
+ pages.add(page);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 5cb11c2..1bde0a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cacheable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -68,7 +68,7 @@ import org.apache.carbondata.core.util.DataTypeUtil;
/**
* Datamap implementation for blocklet.
*/
-public class BlockletDataMap implements DataMap, Cacheable {
+public class BlockletDataMap extends AbstractCoarseGrainDataMap implements Cacheable {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMap.class.getName());
@@ -523,7 +523,6 @@ public class BlockletDataMap implements DataMap, Cacheable {
startIndex++;
}
}
-
return blocklets;
}
@@ -774,4 +773,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
return memoryUsed;
}
+ public SegmentProperties getSegmentProperties() {
+ return segmentProperties;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 61e5ceb..49ed5e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -27,9 +27,10 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -39,9 +40,6 @@ import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -53,7 +51,8 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
+ implements BlockletDetailsFetcher,
SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
@@ -61,10 +60,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
- // segmentId -> SegmentProperties.
- private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
-
- private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
+ private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainDataMap> cache;
@Override
public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
@@ -74,12 +70,12 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public DataMapWriter createWriter(String segmentId) {
+ public AbstractDataMapWriter createWriter(String segmentId, String dataWriterPath) {
throw new UnsupportedOperationException("not implemented");
}
@Override
- public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId) throws IOException {
List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
getTableBlockIndexUniqueIdentifiers(segmentId);
return cache.getAll(tableBlockIndexUniqueIdentifiers);
@@ -140,17 +136,18 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
Blocklet blocklet) throws IOException {
- String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getPath());
+ String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
if (identifier.getCarbonIndexFileName().equals(carbonIndexFileName)) {
DataMap dataMap = cache.get(identifier);
return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
}
}
- throw new IOException("Blocklet with blockid " + blocklet.getPath() + " not found ");
+ throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
}
+
@Override
public List<DataMapDistributable> toDistributable(String segmentId) {
CarbonFile[] carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentId);
@@ -179,7 +176,6 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
- segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -200,7 +196,8 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
if (mapDistributable.getFilePath().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
@@ -217,7 +214,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
indexFile));
}
}
- List<DataMap> dataMaps;
+ List<AbstractCoarseGrainDataMap> dataMaps;
try {
dataMaps = cache.getAll(identifiers);
} catch (IOException e) {
@@ -233,23 +230,20 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
- SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
- if (segmentProperties == null) {
- int[] columnCardinality;
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- List<DataFileFooter> indexInfo =
- fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
- for (DataFileFooter fileFooter : indexInfo) {
- List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
- if (segmentProperties == null) {
- columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
- segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
- }
- }
- segmentPropertiesMap.put(segmentId, segmentProperties);
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ assert (dataMaps.size() > 0);
+ AbstractCoarseGrainDataMap coarseGrainDataMap = dataMaps.get(0);
+ assert (coarseGrainDataMap instanceof BlockletDataMap);
+ BlockletDataMap dataMap = (BlockletDataMap) coarseGrainDataMap;
+ return dataMap.getSegmentProperties();
+ }
+
+ @Override public List<Blocklet> getAllBlocklets(String segmentId) throws IOException {
+ List<Blocklet> blocklets = new ArrayList<>();
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ for (AbstractCoarseGrainDataMap dataMap : dataMaps) {
+ blocklets.addAll(dataMap.prune(null, getSegmentProperties(segmentId), null));
}
- return segmentProperties;
+ return blocklets;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 42505ad..516c460 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -30,7 +31,9 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* wrapper for blocklet data map data
@@ -45,6 +48,8 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
private BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
+ private BlockletSerializer blockletSerializer;
+
public BlockletDataRefNodeWrapper(List<TableBlockInfo> blockInfos, int index,
int[] dimensionLens) {
this.blockInfos = blockInfos;
@@ -69,6 +74,7 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
}
this.index = index;
this.dimensionLens = dimensionLens;
+ this.blockletSerializer = new BlockletSerializer();
}
@Override public DataRefNode getNextDataRefNode() {
@@ -144,15 +150,33 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
this.deleteDeltaDataCache = deleteDeltaDataCache;
}
- @Override public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
+ @Override
+ public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
return deleteDeltaDataCache;
}
- @Override public int numberOfPages() {
+ @Override
+ public int numberOfPages() {
return blockInfos.get(index).getDetailInfo().getPagesCount();
}
- @Override public int getPageRowCount(int pageNumber) {
+ @Override
+ public BitSetGroup getIndexedData() {
+ String dataMapWriterPath = blockInfos.get(index).getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ try {
+ FineGrainBlocklet blocklet = blockletSerializer.deserializeBlocklet(dataMapWriterPath);
+ return blocklet.getBitSetGroup(numberOfPages());
+ } catch (IOException e) {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public int getPageRowCount(int pageNumber) {
return blockInfos.get(index).getDetailInfo().getBlockletInfo()
.getNumberOfRowsPerPage()[pageNumber];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
index b8cffc6..2720700 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -22,6 +22,8 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -31,7 +33,10 @@ import org.apache.carbondata.core.util.CarbonUtil;
*/
public class IndexWrapper extends AbstractIndex {
+ private List<TableBlockInfo> blockInfos;
+
public IndexWrapper(List<TableBlockInfo> blockInfos) {
+ this.blockInfos = blockInfos;
DataFileFooter fileFooter = null;
try {
fileFooter = CarbonUtil.readMetadatFile(blockInfos.get(0));
@@ -46,4 +51,17 @@ public class IndexWrapper extends AbstractIndex {
@Override public void buildIndex(List<DataFileFooter> footerList) {
}
+
+ @Override public void clear() {
+ super.clear();
+ if (blockInfos != null) {
+ for (TableBlockInfo blockInfo : blockInfos) {
+ String dataMapWriterPath = blockInfo.getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ CarbonFile file = FileFactory.getCarbonFile(dataMapWriterPath);
+ FileFactory.deleteAllCarbonFilesOfDir(file);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
deleted file mode 100644
index 9d77010..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.schema;
-
-/**
- * Types of filters of select query
- */
-public enum FilterType {
- EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_EQUAL, LESS_THAN_EQUAL, LIKE
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index f680579..886b12b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -313,6 +313,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
byte[][] filterKeys = dimColumnExecuterInfo.getExcludeFilterKeys();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index fe1421c..0385c73 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -332,6 +332,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet(numberOfRows);
byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
int compareResult = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 60090d0..0ed9137 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -50,11 +50,6 @@ public class BlocksChunkHolder {
private BitSetGroup bitSetGroup;
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- }
-
public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock,
FileHolder fileReader) {
dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index abfc5f4..6e8076e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -153,6 +153,8 @@ public class FilterScanner extends AbstractBlockletScanner {
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
totalBlockletStatistic.getCount() + 1);
+ // set the indexed data if it has any during fgdatamap pruning.
+ blocksChunkHolder.setBitSetGroup(blocksChunkHolder.getDataBlock().getIndexedData());
// apply filter on actual data
BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
// if indexes is empty then return with empty result
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index bd01772..68efdb8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -53,6 +54,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
@@ -96,6 +98,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
@@ -128,6 +131,13 @@ public final class CarbonUtil {
private static final Configuration conf = new Configuration(true);
+ /**
+ * dfs.bytes-per-checksum
+ * HDFS checksum length, block size for a file should be exactly divisible
+ * by this value
+ */
+ private static final int HDFS_CHECKSUM_LENGTH = 512;
+
private CarbonUtil() {
}
@@ -2314,5 +2324,92 @@ public final class CarbonUtil {
return false;
}
+
+ /**
+ * This method will copy the given file to carbon store location
+ *
+ * @param localFilePath local file name with full path
+ * @throws CarbonDataWriterException
+ */
+ public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ String carbonDataDirectoryPath, long fileSizeInBytes)
+ throws CarbonDataWriterException {
+ long copyStartTime = System.currentTimeMillis();
+ LOGGER.info("Copying " + localFilePath + " --> " + carbonDataDirectoryPath);
+ try {
+ CarbonFile localCarbonFile =
+ FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
+ String carbonFilePath = carbonDataDirectoryPath + localFilePath
+ .substring(localFilePath.lastIndexOf(File.separator));
+ copyLocalFileToCarbonStore(carbonFilePath, localFilePath,
+ CarbonCommonConstants.BYTEBUFFER_SIZE,
+ getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
+ } catch (IOException e) {
+ throw new CarbonDataWriterException(
+ "Problem while copying file from local store to carbon store", e);
+ }
+ LOGGER.info(
+ "Total copy time (ms) to copy file " + localFilePath + " is " + (System.currentTimeMillis()
+ - copyStartTime));
+ }
+
+ /**
+ * This method will read the local carbon data file and write to carbon data file in HDFS
+ *
+ * @param carbonStoreFilePath
+ * @param localFilePath
+ * @param bufferSize
+ * @param blockSize
+ * @throws IOException
+ */
+ private static void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
+ int bufferSize, long blockSize) throws IOException {
+ DataOutputStream dataOutputStream = null;
+ DataInputStream dataInputStream = null;
+ try {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
+ + " (bytes");
+ }
+ dataOutputStream = FileFactory
+ .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
+ bufferSize, blockSize);
+ dataInputStream = FileFactory
+ .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
+ IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
+ } finally {
+ CarbonUtil.closeStream(dataInputStream);
+ CarbonUtil.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * This method will return max of block size and file size
+ *
+ * @param blockSize
+ * @param fileSize
+ * @return
+ */
+ private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
+ long maxSize = blockSize;
+ if (fileSize > blockSize) {
+ maxSize = fileSize;
+ }
+ // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
+ // per checksum, dfs.bytes-per-checksum=512 must divide block size
+ long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
+ if (remainder > 0) {
+ maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
+ }
+ // convert to make block size more readable.
+ String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
+ String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
+ String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
+ LOGGER.info(
+ "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
+ + readableFileSize + ", choose the max value " + readableMaxSize
+ + " as the block size on HDFS");
+ return maxSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
index 2ad6327..605796f 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -28,7 +28,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
@@ -47,7 +48,7 @@ import com.google.gson.Gson;
/**
* Datamap implementation for min max blocklet.
*/
-public class MinMaxDataMap implements DataMap {
+public class MinMaxDataMap extends AbstractCoarseGrainDataMap {
private static final LogService LOGGER =
LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
@@ -58,8 +59,8 @@ public class MinMaxDataMap implements DataMap {
private MinMaxIndexBlockDetails[] readMinMaxDataMap;
- @Override public void init(String filePath) throws MemoryException, IOException {
- this.filePath = filePath;
+ @Override public void init(DataMapModel model) throws MemoryException, IOException {
+ this.filePath = model.getFilePath();
CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
for (int i = 0; i < listFiles.length; i++) {
readMinMaxDataMap = readJson(listFiles[i].getPath());
@@ -90,8 +91,7 @@ public class MinMaxDataMap implements DataMap {
return null;
}
dataInputStream = fileOperation.openForRead();
- inStream = new InputStreamReader(dataInputStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ inStream = new InputStreamReader(dataInputStream, "UTF-8");
buffReader = new BufferedReader(inStream);
readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
} catch (IOException e) {
@@ -115,8 +115,7 @@ public class MinMaxDataMap implements DataMap {
if (filterExp == null) {
for (int i = 0; i < readMinMaxDataMap.length; i++) {
- blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
- String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
}
} else {
FilterExecuter filterExecuter =
@@ -126,7 +125,7 @@ public class MinMaxDataMap implements DataMap {
BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
readMinMaxDataMap[startIndex].getMinValues());
if (!bitSet.isEmpty()) {
- blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ blocklets.add(new Blocklet(filePath,
String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
}
startIndex++;
@@ -135,8 +134,7 @@ public class MinMaxDataMap implements DataMap {
return blocklets;
}
- @Override
- public void clear() {
+ @Override public void clear() {
readMinMaxDataMap = null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index b196d0d..5203cb3 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -25,49 +25,51 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.events.ChangeEvent;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxDataMapFactory implements DataMapFactory {
+public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
this.identifier = identifier;
}
/**
* createWriter will return the MinMaxDataWriter.
+ *
* @param segmentId
* @return
*/
- @Override
- public DataMapWriter createWriter(String segmentId) {
- return new MinMaxDataWriter();
+ @Override public AbstractDataMapWriter createWriter(String segmentId, String dataWritePath) {
+ return new MinMaxDataWriter(identifier, segmentId, dataWritePath);
}
/**
* getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ *
* @param segmentId
* @return
* @throws IOException
*/
- @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
- List<DataMap> dataMapList = new ArrayList<>();
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId)
+ throws IOException {
+ List<AbstractCoarseGrainDataMap> dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxDataMap.
MinMaxDataMap dataMap = new MinMaxDataMap();
try {
- dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ dataMap.init(new DataMapModel(
+ identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator));
} catch (MemoryException ex) {
}
@@ -76,7 +78,6 @@ public class MinMaxDataMapFactory implements DataMapFactory {
}
/**
- *
* @param segmentId
* @return
*/
@@ -86,6 +87,7 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clear the DataMap.
+ *
* @param segmentId
*/
@Override public void clear(String segmentId) {
@@ -94,21 +96,20 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clearing the data map.
*/
- @Override
- public void clear() {
+ @Override public void clear() {
}
- @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
return null;
}
- @Override
- public void fireEvent(ChangeEvent event) {
+ @Override public void fireEvent(Event event) {
}
- @Override
- public DataMapMeta getMeta() {
- return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ @Override public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")),
+ new ArrayList<ExpressionType>());
}
}
\ No newline at end of file
[18/50] [abbrv] carbondata git commit: [CARBONDATA-1880] Combine
input small files for GLOBAL_SORT
Posted by ja...@apache.org.
[CARBONDATA-1880] Combine input small files for GLOBAL_SORT
Combine input small files for GLOBAL_SORT to avoid carbon small file issue
This closes #1669
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/694ee774
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/694ee774
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/694ee774
Branch: refs/heads/fgdatamap
Commit: 694ee774cb8ca55ee24cd906368c3cf9cc96b0eb
Parents: 28c9418
Author: QiangCai <qi...@qq.com>
Authored: Fri Dec 15 22:02:28 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Dec 20 14:58:37 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +
.../carbondata/core/util/CarbonProperties.java | 14 ++
.../hadoop/CarbonMultiBlockSplit.java | 17 ++-
...ompactionSupportGlobalSortFunctionTest.scala | 4 +-
...mpactionSupportGlobalSortParameterTest.scala | 5 -
.../MajorCompactionIgnoreInMinorTest.scala | 16 +-
.../dataload/TestGlobalSortDataLoad.scala | 33 ++++-
.../testsuite/datamap/DataMapWriterSuite.scala | 1 +
.../load/DataLoadProcessBuilderOnSpark.scala | 148 ++++++++++++++++---
.../load/DataLoadProcessorStepOnSpark.scala | 7 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 121 +++++++++++----
.../apache/spark/sql/util/CarbonException.scala | 8 +-
.../apache/spark/sql/util/SparkSQLUtil.scala | 25 ++++
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
15 files changed, 333 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f67b0c5..9534099 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1277,6 +1277,10 @@ public final class CarbonCommonConstants {
public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION = "carbon.custom.block.distribution";
public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT = "false";
+ @CarbonProperty
+ public static final String CARBON_COMBINE_SMALL_INPUT_FILES = "carbon.mergeSmallFileRead.enable";
+ public static final String CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT = "false";
+
public static final int DICTIONARY_DEFAULT_CARDINALITY = 1;
@CarbonProperty
public static final String SPARK_SCHEMA_STRING_LENGTH_THRESHOLD =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index fe396cb..11aea99 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -106,6 +106,7 @@ public final class CarbonProperties {
validateLockType();
validateCarbonCSVReadBufferSizeByte();
validateHandoffSize();
+ validateCombineSmallInputFiles();
}
private void validateCarbonCSVReadBufferSizeByte() {
@@ -205,6 +206,19 @@ public final class CarbonProperties {
}
}
+ private void validateCombineSmallInputFiles() {
+ String combineSmallInputFilesStr =
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES);
+ boolean isValidBooleanValue = CarbonUtil.validateBoolean(combineSmallInputFilesStr);
+ if (!isValidBooleanValue) {
+ LOGGER.warn("The combine small files value \"" + combineSmallInputFilesStr
+ + "\" is invalid. Using the default value \""
+ + CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
+ carbonProperties.setProperty(CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES,
+ CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
+ }
+ }
+
private void validateEnableUnsafeSort() {
String unSafeSortStr = carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT);
boolean isValidBooleanValue = CarbonUtil.validateBoolean(unSafeSortStr);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
index 96fe909..aed3449 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
@@ -47,15 +47,19 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ private long length;
+
public CarbonMultiBlockSplit() {
splitList = null;
locations = null;
+ length = 0;
}
public CarbonMultiBlockSplit(AbsoluteTableIdentifier identifier, List<CarbonInputSplit> splitList,
String[] locations) throws IOException {
this.splitList = splitList;
this.locations = locations;
+ calculateLength();
}
public CarbonMultiBlockSplit(AbsoluteTableIdentifier identifier, List<CarbonInputSplit> splitList,
@@ -63,6 +67,7 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
this.splitList = splitList;
this.locations = locations;
this.fileFormat = fileFormat;
+ calculateLength();
}
/**
@@ -75,11 +80,19 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
@Override
public long getLength() throws IOException, InterruptedException {
+ return length;
+ }
+
+ public void setLength(long length) {
+ this.length = length;
+ }
+
+ private void calculateLength() {
long total = 0;
- for (InputSplit split: splitList) {
+ for (CarbonInputSplit split : splitList) {
total += split.getLength();
}
- return total;
+ length = total;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
index 4958f55..9014edb 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
@@ -439,7 +439,7 @@ class CompactionSupportGlobalSortFunctionTest extends QueryTest with BeforeAndAf
}
sql("ALTER TABLE compaction_globalsort COMPACT 'MAJOR'")
- assert(getIndexFileCount("compaction_globalsort", "0.1") === 3)
+ assert(getIndexFileCount("compaction_globalsort", "0.1") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM compaction_globalsort"), Seq(Row(72)))
checkAnswer(sql("SELECT * FROM compaction_globalsort order by name, id"),
sql("SELECT * FROM carbon_localsort order by name, id"))
@@ -454,7 +454,7 @@ class CompactionSupportGlobalSortFunctionTest extends QueryTest with BeforeAndAf
}
sql("ALTER TABLE compaction_globalsort COMPACT 'MINOR'")
- assert(getIndexFileCount("compaction_globalsort", "0.1") === 3)
+ assert(getIndexFileCount("compaction_globalsort", "0.1") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM compaction_globalsort"), Seq(Row(72)))
checkAnswer(sql("SELECT * FROM compaction_globalsort order by name, id"),
sql("SELECT * FROM carbon_localsort order by name, id"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
index f9959fa..02c602a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
@@ -17,8 +17,6 @@
package org.apache.carbondata.spark.testsuite.datacompaction
-import java.io.{File, FilenameFilter}
-
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.Row
@@ -26,10 +24,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
-import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
class CompactionSupportGlobalSortParameterTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
val filePath: String = s"$resourcesPath/globalsort"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index ed63fdf..61de615 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -59,16 +59,16 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
)
// compaction will happen here.
- sql("alter table ignoremajor compact 'major'"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor OPTIONS" +
+ sql("alter table ignoremajor compact 'major'")
+
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE ignoremajor OPTIONS" +
+ )
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE ignoremajor OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
- )
- sql("alter table ignoremajor compact 'minor'"
- )
+ )
+ sql("alter table ignoremajor compact 'minor'"
+ )
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 6bbc763..9ce9675 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -17,6 +17,10 @@
package org.apache.carbondata.spark.testsuite.dataload
+import java.io.{File, FileWriter}
+
+import org.apache.commons.io.FileUtils
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -105,7 +109,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_globalsort " +
"OPTIONS('BAD_RECORDS_ACTION'='REDIRECT')")
- assert(getIndexFileCount("carbon_globalsort") === 3)
+ assert(getIndexFileCount("carbon_globalsort") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM carbon_globalsort"), Seq(Row(11)))
}
@@ -115,7 +119,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_globalsort " +
"OPTIONS('SINGLE_PASS'='TRUE')")
- assert(getIndexFileCount("carbon_globalsort") === 3)
+ assert(getIndexFileCount("carbon_globalsort") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM carbon_globalsort"), Seq(Row(12)))
checkAnswer(sql("SELECT * FROM carbon_globalsort ORDER BY name"),
sql("SELECT * FROM carbon_localsort_once ORDER BY name"))
@@ -164,7 +168,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_globalsort")
sql("ALTER TABLE carbon_globalsort COMPACT 'MAJOR'")
- assert(getIndexFileCount("carbon_globalsort") === 3)
+ assert(getIndexFileCount("carbon_globalsort") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM carbon_globalsort"), Seq(Row(24)))
checkAnswer(sql("SELECT * FROM carbon_globalsort ORDER BY name, id"),
sql("SELECT * FROM carbon_localsort_twice ORDER BY name, id"))
@@ -223,7 +227,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_globalsort")
sql("DELETE FROM carbon_globalsort WHERE id = 1").show
- assert(getIndexFileCount("carbon_globalsort") === 3)
+ assert(getIndexFileCount("carbon_globalsort") === 2)
checkAnswer(sql("SELECT COUNT(*) FROM carbon_globalsort"), Seq(Row(11)))
checkAnswer(sql("SELECT * FROM carbon_globalsort ORDER BY name, id"),
sql("SELECT * FROM carbon_localsort_delete ORDER BY name, id"))
@@ -250,6 +254,27 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql("SELECT * FROM carbon_localsort_update ORDER BY name, id"))
}
+ test("LOAD with small files") {
+ val inputPath = new File("target/small_files").getCanonicalPath
+ val folder = new File(inputPath)
+ if (folder.exists()) {
+ FileUtils.deleteDirectory(folder)
+ }
+ folder.mkdir()
+ for (i <- 0 to 100) {
+ val file = s"$folder/file$i.csv"
+ val writer = new FileWriter(file)
+ writer.write("id,name,city,age\n")
+ writer.write(s"$i,name_$i,city_$i,${ i % 100 }")
+ writer.close()
+ }
+ sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE carbon_globalsort")
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
+ val segmentDir = carbonTablePath.getSegmentDir("0", "0")
+ assertResult(5)(new File(segmentDir).listFiles().length)
+ }
+
// ----------------------------------- INSERT INTO -----------------------------------
test("INSERT INTO") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOAD_SORT_SCOPE, "GLOBAL_SORT")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index c137fc7..f73a202 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -70,6 +70,7 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
sqlContext.sparkContext.parallelize(1 to numRows)
.map(x => ("a", "b", x))
.toDF("c1", "c2", "c3")
+ .sort("c3")
}
def dropTable(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index c14e0a7..2537a0c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -17,17 +17,26 @@
package org.apache.carbondata.spark.load
-import java.util.Comparator
+import java.text.SimpleDateFormat
+import java.util.{Comparator, Date, Locale}
+
+import scala.collection.mutable.ArrayBuffer
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.io.NullWritable
+import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.spark.{SparkContext, TaskContext}
+import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
+import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
+import org.apache.hadoop.mapreduce.task.{JobContextImpl, TaskAttemptContextImpl}
+import org.apache.spark.TaskContext
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.rdd.NewHadoopRDD
-import org.apache.spark.sql.DataFrame
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.command.ExecutionErrors
+import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile}
+import org.apache.spark.sql.util.SparkSQLUtil.sessionState
import org.apache.spark.storage.StorageLevel
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -36,10 +45,11 @@ import org.apache.carbondata.core.datastore.row.CarbonRow
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.loading.{DataLoadProcessBuilder, FailureCauses}
-import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.sort.sortdata.{NewRowComparator, NewRowComparatorForNormalDims, SortParameters}
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
+import org.apache.carbondata.spark.rdd.SerializableConfiguration
import org.apache.carbondata.spark.util.CommonUtil
/**
@@ -49,7 +59,7 @@ object DataLoadProcessBuilderOnSpark {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
def loadDataUsingGlobalSort(
- sc: SparkContext,
+ sparkSession: SparkSession,
dataFrame: Option[DataFrame],
model: CarbonLoadModel,
hadoopConf: Configuration): Array[(String, (LoadMetadataDetails, ExecutionErrors))] = {
@@ -57,21 +67,13 @@ object DataLoadProcessBuilderOnSpark {
dataFrame.get.rdd
} else {
// input data from files
- CommonUtil.configureCSVInputFormat(hadoopConf, model)
- hadoopConf.set(FileInputFormat.INPUT_DIR, model.getFactFilePath)
val columnCount = model.getCsvHeaderColumns.length
- val jobConf = new JobConf(hadoopConf)
- SparkHadoopUtil.get.addCredentials(jobConf)
- new NewHadoopRDD[NullWritable, StringArrayWritable](
- sc,
- classOf[CSVInputFormat],
- classOf[NullWritable],
- classOf[StringArrayWritable],
- jobConf)
- .map(x => DataLoadProcessorStepOnSpark.toStringArrayRow(x._2, columnCount))
+ csvFileScanRDD(sparkSession, model, hadoopConf)
+ .map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
}
model.setPartitionId("0")
+ val sc = sparkSession.sparkContext
val modelBroadcast = sc.broadcast(model)
val partialSuccessAccum = sc.accumulator(0, "Partial Success Accumulator")
@@ -160,4 +162,112 @@ object DataLoadProcessBuilderOnSpark {
Array((uniqueLoadStatusId, (loadMetadataDetails, executionErrors)))
}
}
+
+ /**
+ * creates a RDD that does reading of multiple CSV files
+ */
+ def csvFileScanRDD(
+ spark: SparkSession,
+ model: CarbonLoadModel,
+ hadoopConf: Configuration
+ ): RDD[InternalRow] = {
+ // 1. partition
+ val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
+ val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
+ val defaultParallelism = spark.sparkContext.defaultParallelism
+ CommonUtil.configureCSVInputFormat(hadoopConf, model)
+ hadoopConf.set(FileInputFormat.INPUT_DIR, model.getFactFilePath)
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ val jobContext = new JobContextImpl(jobConf, null)
+ val inputFormat = new CSVInputFormat()
+ val rawSplits = inputFormat.getSplits(jobContext).toArray
+ val splitFiles = rawSplits.map { split =>
+ val fileSplit = split.asInstanceOf[FileSplit]
+ PartitionedFile(
+ InternalRow.empty,
+ fileSplit.getPath.toString,
+ fileSplit.getStart,
+ fileSplit.getLength,
+ fileSplit.getLocations)
+ }.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
+ val totalBytes = splitFiles.map(_.length + openCostInBytes).sum
+ val bytesPerCore = totalBytes / defaultParallelism
+
+ val maxSplitBytes = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
+ LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
+ s"open cost is considered as scanning $openCostInBytes bytes.")
+
+ val partitions = new ArrayBuffer[FilePartition]
+ val currentFiles = new ArrayBuffer[PartitionedFile]
+ var currentSize = 0L
+
+ def closePartition(): Unit = {
+ if (currentFiles.nonEmpty) {
+ val newPartition =
+ FilePartition(
+ partitions.size,
+ currentFiles.toArray.toSeq)
+ partitions += newPartition
+ }
+ currentFiles.clear()
+ currentSize = 0
+ }
+
+ splitFiles.foreach { file =>
+ if (currentSize + file.length > maxSplitBytes) {
+ closePartition()
+ }
+ // Add the given file to the current partition.
+ currentSize += file.length + openCostInBytes
+ currentFiles += file
+ }
+ closePartition()
+
+ // 2. read function
+ val serializableConfiguration = new SerializableConfiguration(jobConf)
+ val readFunction = new (PartitionedFile => Iterator[InternalRow]) with Serializable {
+ override def apply(file: PartitionedFile): Iterator[InternalRow] = {
+ new Iterator[InternalRow] {
+ val hadoopConf = serializableConfiguration.value
+ val jobTrackerId: String = {
+ val formatter = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
+ formatter.format(new Date())
+ }
+ val attemptId = new TaskAttemptID(jobTrackerId, 0, TaskType.MAP, 0, 0)
+ val hadoopAttemptContext = new TaskAttemptContextImpl(hadoopConf, attemptId)
+ val inputSplit =
+ new FileSplit(new Path(file.filePath), file.start, file.length, file.locations)
+ var finished = false
+ val inputFormat = new CSVInputFormat()
+ val reader = inputFormat.createRecordReader(inputSplit, hadoopAttemptContext)
+ reader.initialize(inputSplit, hadoopAttemptContext)
+
+ override def hasNext: Boolean = {
+ if (!finished) {
+ if (reader != null) {
+ if (reader.nextKeyValue()) {
+ true
+ } else {
+ finished = true
+ reader.close()
+ false
+ }
+ } else {
+ finished = true
+ false
+ }
+ } else {
+ false
+ }
+ }
+
+ override def next(): InternalRow = {
+ new GenericInternalRow(reader.getCurrentValue.get().asInstanceOf[Array[Any]])
+ }
+ }
+ }
+ }
+ new FileScanRDD(spark, readFunction, partitions)
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 5e6ba98..154d3ed 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -23,6 +23,8 @@ import com.univocity.parsers.common.TextParsingException
import org.apache.spark.{Accumulator, SparkEnv, TaskContext}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException
@@ -30,7 +32,6 @@ import org.apache.carbondata.core.datastore.row.CarbonRow
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.loading.{DataLoadProcessBuilder, TableProcessingOperations}
import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.parser.impl.RowParserImpl
@@ -45,9 +46,9 @@ import org.apache.carbondata.spark.util.Util
object DataLoadProcessorStepOnSpark {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- def toStringArrayRow(row: StringArrayWritable, columnCount: Int): StringArrayRow = {
+ def toStringArrayRow(row: InternalRow, columnCount: Int): StringArrayRow = {
val outRow = new StringArrayRow(new Array[String](columnCount))
- outRow.setValues(row.get())
+ outRow.setValues(row.asInstanceOf[GenericInternalRow].values.asInstanceOf[Array[String]])
}
def toRDDIterator(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index e58bfd4..09dbd71 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -31,6 +31,8 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.hive.DistributionUtil
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.util.SparkSQLUtil.sessionState
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -55,7 +57,7 @@ import org.apache.carbondata.spark.util.SparkDataTypeConverterImpl
* level filtering in driver side.
*/
class CarbonScanRDD(
- @transient sc: SparkContext,
+ @transient spark: SparkSession,
val columnProjection: CarbonProjection,
var filterExpression: Expression,
identifier: AbsoluteTableIdentifier,
@@ -63,7 +65,7 @@ class CarbonScanRDD(
@transient tableInfo: TableInfo,
inputMetricsStats: InitInputMetrics,
@transient val partitionNames: Seq[String])
- extends CarbonRDDWithTableInfo[InternalRow](sc, Nil, serializedTableInfo) {
+ extends CarbonRDDWithTableInfo[InternalRow](spark.sparkContext, Nil, serializedTableInfo) {
private val queryId = sparkContext.getConf.get("queryId", System.nanoTime() + "")
private val jobTrackerId: String = {
@@ -186,31 +188,74 @@ class CarbonScanRDD(
}
}
noOfNodes = nodeBlockMapping.size
- } else {
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION,
- CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT).toBoolean) {
- // Use blocklet distribution
- // Randomize the blocklets for better shuffling
- Random.shuffle(splits.asScala).zipWithIndex.foreach { splitWithIndex =>
- val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
- Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
- splitWithIndex._1.getLocations)
- val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
- result.add(partition)
+ } else if (CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT).toBoolean) {
+ // Use blocklet distribution
+ // Randomize the blocklets for better shuffling
+ Random.shuffle(splits.asScala).zipWithIndex.foreach { splitWithIndex =>
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier,
+ Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
+ splitWithIndex._1.getLocations)
+ val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+ result.add(partition)
+ }
+ } else if (CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES,
+ CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT).toBoolean) {
+
+ // sort blocks in reverse order of length
+ val blockSplits = splits
+ .asScala
+ .map(_.asInstanceOf[CarbonInputSplit])
+ .groupBy(f => f.getBlockPath)
+ .map { blockSplitEntry =>
+ new CarbonMultiBlockSplit(identifier,
+ blockSplitEntry._2.asJava,
+ blockSplitEntry._2.flatMap(f => f.getLocations).distinct.toArray)
+ }.toArray.sortBy(_.getLength)(implicitly[Ordering[Long]].reverse)
+
+ val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
+ val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
+ val defaultParallelism = spark.sparkContext.defaultParallelism
+ val totalBytes = blockSplits.map(_.getLength + openCostInBytes).sum
+ val bytesPerCore = totalBytes / defaultParallelism
+
+ val maxSplitBytes = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
+ LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
+ s"open cost is considered as scanning $openCostInBytes bytes.")
+
+ val currentFiles = new ArrayBuffer[CarbonMultiBlockSplit]
+ var currentSize = 0L
+
+ def closePartition(): Unit = {
+ if (currentFiles.nonEmpty) {
+ result.add(combineSplits(currentFiles, currentSize, result.size()))
}
- } else {
- // Use block distribution
- splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).
- groupBy(f => f.getBlockPath).values.zipWithIndex.foreach { splitWithIndex =>
- val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
- splitWithIndex._1.asJava,
- splitWithIndex._1.flatMap(f => f.getLocations).distinct.toArray)
- val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
- result.add(partition)
+ currentFiles.clear()
+ currentSize = 0
+ }
+
+ blockSplits.foreach { file =>
+ if (currentSize + file.getLength > maxSplitBytes) {
+ closePartition()
}
+ // Add the given file to the current partition.
+ currentSize += file.getLength + openCostInBytes
+ currentFiles += file
+ }
+ closePartition()
+ } else {
+ // Use block distribution
+ splits.asScala.map(_.asInstanceOf[CarbonInputSplit])
+ .groupBy(f => f.getBlockPath).values.zipWithIndex.foreach { splitWithIndex =>
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier,
+ splitWithIndex._1.asJava,
+ splitWithIndex._1.flatMap(f => f.getLocations).distinct.toArray)
+ val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+ result.add(partition)
}
}
@@ -232,6 +277,32 @@ class CarbonScanRDD(
result.asScala
}
+ def combineSplits(
+ splits: ArrayBuffer[CarbonMultiBlockSplit],
+ size: Long,
+ partitionId: Int
+ ): CarbonSparkPartition = {
+ val carbonInputSplits = splits.flatMap(_.getAllSplits.asScala)
+
+ // Computes total number of bytes can be retrieved from each host.
+ val hostToNumBytes = mutable.HashMap.empty[String, Long]
+ splits.foreach { split =>
+ split.getLocations.filter(_ != "localhost").foreach { host =>
+ hostToNumBytes(host) = hostToNumBytes.getOrElse(host, 0L) + split.getLength
+ }
+ }
+ // Takes the first 3 hosts with the most data to be retrieved
+ val locations = hostToNumBytes
+ .toSeq
+ .sortBy(_._2)(implicitly[Ordering[Long]].reverse)
+ .take(3)
+ .map(_._1)
+ .toArray
+
+ val multiBlockSplit = new CarbonMultiBlockSplit(null, carbonInputSplits.asJava, locations)
+ new CarbonSparkPartition(id, partitionId, multiBlockSplit)
+ }
+
override def internalCompute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
val queryStartTime = System.currentTimeMillis
val carbonPropertiesFilePath = System.getProperty("carbon.properties.filepath", null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
index 9fd7099..7dabddb 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
@@ -17,8 +17,8 @@
package org.apache.spark.sql.util
- import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.AnalysisException
- object CarbonException {
- def analysisException(message: String): Nothing = throw new AnalysisException(message)
- }
+object CarbonException {
+ def analysisException(message: String): Nothing = throw new AnalysisException(message)
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
new file mode 100644
index 0000000..370f80c
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.SessionState
+
+object SparkSQLUtil {
+ def sessionState(sparkSession: SparkSession): SessionState = sparkSession.sessionState
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 0b786b5..72c979a 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -342,7 +342,7 @@ object CarbonDataRDDFactory {
status = if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null) {
loadDataForPartitionTable(sqlContext, dataFrame, carbonLoadModel, hadoopConf)
} else if (isSortTable && sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
- DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(sqlContext.sparkContext,
+ DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(sqlContext.sparkSession,
dataFrame, carbonLoadModel, hadoopConf)
} else if (dataFrame.isDefined) {
loadDataFrame(sqlContext, dataFrame, carbonLoadModel)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/694ee774/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 148fca8..ca0c51d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -76,7 +76,7 @@ case class CarbonDatasourceHadoopRelation(
requiredColumns.foreach(projection.addColumn)
val inputMetricsStats: CarbonInputMetrics = new CarbonInputMetrics
new CarbonScanRDD(
- sparkSession.sparkContext,
+ sparkSession,
projection,
filterExpression.orNull,
identifier,
[22/50] [abbrv] carbondata git commit: [CARBONDATA-1913][global_sort]
Global Sort data dataload fails for big data with RPC timeout exception
Posted by ja...@apache.org.
[CARBONDATA-1913][global_sort] Global Sort data dataload fails for big data with RPC timeout exception
When gloabl sort option is used for big data then for some times it fails with RPC timeout after 120s.This is happening because the driver is not able to unpersist rdd cache with in 120s.The issue is happening due to rdd unpersist blocking call. Sometimes spark is not ableto unppersist the rdd in default spark.rpc.askTimeout or spark.network.timeout time.Clean cache only if persisted and keeping unpersist non-blocking. Non blocking will not wait for cache clean up task to finish and non-blocking call will not have any functional impact as spark automatically monitors the cache usage on each node and drops out old data partitions in a least-recently used (LRU) fashion.
This closes #1685
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4c9af60d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4c9af60d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4c9af60d
Branch: refs/heads/fgdatamap
Commit: 4c9af60d4af81687ce25bd76cc075c868a5e431c
Parents: 410df07
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Tue Dec 19 19:18:06 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed Dec 20 18:36:52 2017 +0530
----------------------------------------------------------------------
.../spark/load/DataLoadProcessBuilderOnSpark.scala | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9af60d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 2537a0c..781b484 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -136,8 +136,12 @@ object DataLoadProcessBuilderOnSpark {
DataLoadProcessorStepOnSpark.writeFunc(rows, context.partitionId, modelBroadcast,
writeStepRowCounter))
- // clean cache
- convertRDD.unpersist()
+ // clean cache only if persisted and keeping unpersist non-blocking as non-blocking call will
+ // not have any functional impact as spark automatically monitors the cache usage on each node
+ // and drops out old data partiotions in a least-recently used (LRU) fashion.
+ if (numPartitions > 1) {
+ convertRDD.unpersist(false)
+ }
// Log the number of rows in each step
LOGGER.info("Total rows processed in step Input Processor: " + inputStepRowCounter.value)
[13/50] [abbrv] carbondata git commit: [CARBONDATA-1884] Add CTAS
support to carbondata
Posted by ja...@apache.org.
[CARBONDATA-1884] Add CTAS support to carbondata
Implemented CTAS feature in carbondata
This closes #1665
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/59de7cdb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/59de7cdb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/59de7cdb
Branch: refs/heads/fgdatamap
Commit: 59de7cdbd234d921af1a4f535d240943778a775d
Parents: 54eedfe
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Dec 13 21:40:19 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Dec 19 14:43:57 2017 +0800
----------------------------------------------------------------------
.../createTable/TestCreateTableAsSelect.scala | 161 +++++++++++++++++++
.../TestDataWithDicExcludeAndInclude.scala | 11 --
.../spark/util/CarbonReflectionUtils.scala | 2 +-
.../CarbonCreateTableAsSelectCommand.scala | 113 +++++++++++++
.../spark/sql/hive/CarbonFileMetastore.scala | 44 +++++
.../apache/spark/sql/hive/CarbonMetaStore.scala | 28 +++-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 56 +++++--
.../src/main/spark2.1/CarbonSessionState.scala | 26 +--
.../src/main/spark2.2/CarbonSessionState.scala | 26 +--
9 files changed, 417 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
new file mode 100644
index 0000000..ffe6261
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.Spark2TestQueryExecutor
+import org.apache.spark.sql.{CarbonEnv, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+
+/**
+ * test functionality for create table as select command
+ */
+class TestCreateTableAsSelect extends QueryTest with BeforeAndAfterAll {
+
+ private def createTablesAndInsertData {
+ // create carbon table and insert data
+ sql("CREATE TABLE carbon_ctas_test(key INT, value STRING) STORED by 'carbondata'")
+ sql("insert into carbon_ctas_test select 100,'spark'")
+ sql("insert into carbon_ctas_test select 200,'hive'")
+
+ // create parquet table and insert data
+ sql("CREATE TABLE parquet_ctas_test(key INT, value STRING) STORED as parquet")
+ sql("insert into parquet_ctas_test select 100,'spark'")
+ sql("insert into parquet_ctas_test select 200,'hive'")
+
+ // create hive table and insert data
+ sql("CREATE TABLE orc_ctas_test(key INT, value STRING) STORED as ORC")
+ sql("insert into orc_ctas_test select 100,'spark'")
+ sql("insert into orc_ctas_test select 200,'hive'")
+ }
+
+ override def beforeAll {
+ sql("DROP TABLE IF EXISTS carbon_ctas_test")
+ sql("DROP TABLE IF EXISTS parquet_ctas_test")
+ sql("DROP TABLE IF EXISTS orc_ctas_test")
+ createTablesAndInsertData
+ }
+
+ test("test create table as select with select from same table name when table exists") {
+ sql("drop table if exists ctas_same_table_name")
+ sql("CREATE TABLE ctas_same_table_name(key INT, value STRING) STORED by 'carbondata'")
+ intercept[Exception] {
+ sql("create table ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name")
+ }
+ }
+
+ test("test create table as select with select from same table name when table does not exists") {
+ sql("drop table if exists ctas_same_table_name")
+ intercept[Exception] {
+ sql("create table ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name")
+ }
+ }
+
+ test("test create table as select with select from same table name with if not exists clause") {
+ sql("drop table if exists ctas_same_table_name")
+ sql("CREATE TABLE ctas_same_table_name(key INT, value STRING) STORED by 'carbondata'")
+ sql("create table if not exists ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name")
+ assert(true)
+ }
+
+ test("test create table as select with select from another carbon table") {
+ sql("DROP TABLE IF EXISTS ctas_select_carbon")
+ sql("create table ctas_select_carbon stored by 'carbondata' as select * from carbon_ctas_test")
+ checkAnswer(sql("select * from ctas_select_carbon"), sql("select * from carbon_ctas_test"))
+ }
+
+ test("test create table as select with select from another parquet table") {
+ sql("DROP TABLE IF EXISTS ctas_select_parquet")
+ sql("create table ctas_select_parquet stored by 'carbondata' as select * from parquet_ctas_test")
+ checkAnswer(sql("select * from ctas_select_parquet"), sql("select * from parquet_ctas_test"))
+ }
+
+ test("test create table as select with select from another hive/orc table") {
+ sql("DROP TABLE IF EXISTS ctas_select_orc")
+ sql("create table ctas_select_orc stored by 'carbondata' as select * from orc_ctas_test")
+ checkAnswer(sql("select * from ctas_select_orc"), sql("select * from orc_ctas_test"))
+ }
+
+ test("test create table as select with where clause in select from carbon table that returns data") {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon")
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=100")
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test where key=100"))
+ }
+
+ test(
+ "test create table as select with where clause in select from carbon table that does not return data") {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon")
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=300")
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test where key=300"))
+ }
+
+ test("test create table as select with where clause in select from carbon table and load again") {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon")
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=100")
+ sql("insert into ctas_select_where_carbon select 200,'hive'")
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test"))
+ }
+
+ test("test create table as select with where clause in select from parquet table") {
+ sql("DROP TABLE IF EXISTS ctas_select_where_parquet")
+ sql("create table ctas_select_where_parquet stored by 'carbondata' as select * from parquet_ctas_test where key=100")
+ checkAnswer(sql("select * from ctas_select_where_parquet"), sql("select * from parquet_ctas_test where key=100"))
+ }
+
+ test("test create table as select with where clause in select from hive/orc table") {
+ sql("DROP TABLE IF EXISTS ctas_select_where_orc")
+ sql("create table ctas_select_where_orc stored by 'carbondata' as select * from orc_ctas_test where key=100")
+ checkAnswer(sql("select * from ctas_select_where_orc"), sql("select * from orc_ctas_test where key=100"))
+ }
+
+ test("test create table as select with select directly having the data") {
+ sql("DROP TABLE IF EXISTS ctas_select_direct_data")
+ sql("create table ctas_select_direct_data stored by 'carbondata' as select 300,'carbondata'")
+ checkAnswer(sql("select * from ctas_select_direct_data"), Seq(Row(300,"carbondata")))
+ }
+
+ test("test create table as select with TBLPROPERTIES") {
+ sql("DROP TABLE IF EXISTS ctas_tblproperties_test")
+ sql(
+ "create table ctas_tblproperties_test stored by 'carbondata' TBLPROPERTIES" +
+ "('DICTIONARY_INCLUDE'='key', 'sort_scope'='global_sort') as select * from carbon_ctas_test")
+ checkAnswer(sql("select * from ctas_tblproperties_test"), sql("select * from carbon_ctas_test"))
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ .lookupRelation(Option("default"), "ctas_tblproperties_test")(Spark2TestQueryExecutor.spark)
+ .asInstanceOf[CarbonRelation].carbonTable
+ val metadataFolderPath: CarbonFile = FileFactory.getCarbonFile(carbonTable.getMetaDataFilepath)
+ assert(metadataFolderPath.exists())
+ val dictFiles: Array[CarbonFile] = metadataFolderPath.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = {
+ file.getName.contains(".dict") || file.getName.contains(".sortindex")
+ }
+ })
+ assert(dictFiles.length == 3)
+ }
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS carbon_ctas_test")
+ sql("DROP TABLE IF EXISTS parquet_ctas_test")
+ sql("DROP TABLE IF EXISTS orc_ctas_test")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index 484c304..c788857 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -100,17 +100,6 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
).message.contains("Operation not allowed: CREATE EXTERNAL TABLE"))
}
- test("test CTAS should fail") {
- assert(intercept[AnalysisException](
- sql(
- """
- | CREATE TABLE t1 (id string, value int)
- | STORED BY 'carbondata'
- | AS SELECT 'ABC', 1 FROM t2
- """.stripMargin)
- ).message.contains("Operation not allowed: CREATE TABLE AS SELECT"))
- }
-
override def afterAll {
dropTable
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
index ba51077..19b967a 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
@@ -189,7 +189,7 @@ object CarbonReflectionUtils {
createObject(
"org.apache.spark.sql.hive.CarbonSqlAstBuilder",
conf,
- sqlParser)._1.asInstanceOf[AstBuilder]
+ sqlParser, sparkSession)._1.asInstanceOf[AstBuilder]
} else {
throw new UnsupportedOperationException("Spark version not supported")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
new file mode 100644
index 0000000..26a8f6f
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.table
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.command.AtomicRunnableCommand
+import org.apache.spark.sql.execution.command.management.CarbonInsertIntoCommand
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.metadata.schema.table.TableInfo
+
+/**
+ * Create table and insert the query result into it.
+ *
+ * @param tableInfo the Table Describe, which may contains serde, storage handler etc.
+ * @param query the query whose result will be insert into the new relation
+ * @param ifNotExistsSet allow continue working if it's already exists, otherwise
+ * raise exception
+ * @param tableLocation store location where the table need to be created
+ */
+case class CarbonCreateTableAsSelectCommand(
+ tableInfo: TableInfo,
+ query: LogicalPlan,
+ ifNotExistsSet: Boolean = false,
+ tableLocation: Option[String] = None) extends AtomicRunnableCommand {
+
+ /**
+ * variable to be used for insert into command for checking whether the
+ * table is created newly or already existed
+ */
+ var isTableCreated: Boolean = false
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ val tableName = tableInfo.getFactTable.getTableName
+ var databaseOpt: Option[String] = None
+ if (tableInfo.getDatabaseName != null) {
+ databaseOpt = Some(tableInfo.getDatabaseName)
+ }
+ val dbName = CarbonEnv.getDatabaseName(databaseOpt)(sparkSession)
+ LOGGER.audit(s"Request received for CTAS for $dbName.$tableName")
+ // check if table already exists
+ if (sparkSession.sessionState.catalog.listTables(dbName)
+ .exists(_.table.equalsIgnoreCase(tableName))) {
+ if (!ifNotExistsSet) {
+ LOGGER.audit(
+ s"Table creation with Database name [$dbName] and Table name [$tableName] failed. " +
+ s"Table [$tableName] already exists under database [$dbName]")
+ throw new TableAlreadyExistsException(dbName, tableName)
+ }
+ } else {
+ // execute command to create carbon table
+ CarbonCreateTableCommand(tableInfo, ifNotExistsSet, tableLocation).run(sparkSession)
+ isTableCreated = true
+ }
+ Seq.empty
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ if (isTableCreated) {
+ val tableName = tableInfo.getFactTable.getTableName
+ var databaseOpt: Option[String] = None
+ if (tableInfo.getDatabaseName != null) {
+ databaseOpt = Some(tableInfo.getDatabaseName)
+ }
+ val dbName = CarbonEnv.getDatabaseName(databaseOpt)(sparkSession)
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ val carbonDataSourceHadoopRelation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .createCarbonDataSourceHadoopRelation(sparkSession,
+ TableIdentifier(tableName, Option(dbName)))
+ // execute command to load data into carbon table
+ CarbonInsertIntoCommand(
+ carbonDataSourceHadoopRelation,
+ query,
+ overwrite = false,
+ partition = Map.empty).run(sparkSession)
+ LOGGER.audit(s"CTAS operation completed successfully for $dbName.$tableName")
+ }
+ Seq.empty
+ }
+
+ override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
+ val tableName = tableInfo.getFactTable.getTableName
+ var databaseOpt: Option[String] = None
+ if (tableInfo.getDatabaseName != null) {
+ databaseOpt = Some(tableInfo.getDatabaseName)
+ }
+ val dbName = CarbonEnv.getDatabaseName(databaseOpt)(sparkSession)
+ // drop the created table.
+ CarbonDropTableCommand(
+ ifExistsSet = false,
+ Option(dbName), tableName).run(sparkSession)
+ Seq.empty
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index f7a1eed..ba222e2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.hive
+import java.net.URI
import java.util.UUID
import java.util.concurrent.atomic.AtomicLong
@@ -528,4 +529,47 @@ class CarbonFileMetastore extends CarbonMetaStore {
val tableMetadataFile = tablePath.getSchemaFilePath
CarbonUtil.readSchemaFile(tableMetadataFile)
}
+
+ override def createCarbonDataSourceHadoopRelation(
+ sparkSession: SparkSession,
+ tableIdentifier: TableIdentifier): CarbonDatasourceHadoopRelation = {
+ val relation: LogicalPlan = sparkSession.sessionState.catalog.lookupRelation(tableIdentifier)
+ relation match {
+ case SubqueryAlias(_,
+ LogicalRelation(carbonDataSourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _)) =>
+ carbonDataSourceHadoopRelation
+ case LogicalRelation(
+ carbonDataSourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
+ carbonDataSourceHadoopRelation
+ case SubqueryAlias(_, c)
+ if SPARK_VERSION.startsWith("2.2") &&
+ (c.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
+ c.getClass.getName
+ .equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
+ c.getClass.getName.equals(
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation")) =>
+ val catalogTable =
+ CarbonReflectionUtils.getFieldOfCatalogTable("tableMeta", c).asInstanceOf[CatalogTable]
+ catalogTable.provider match {
+ case Some(name) if name.equals("org.apache.spark.sql.CarbonSource") => name
+ case _ =>
+ throw new NoSuchTableException(tableIdentifier.database.get, tableIdentifier.table)
+ }
+ val tableLocation = catalogTable.storage.locationUri match {
+ case tableLoc@Some(uri) =>
+ if (tableLoc.get.isInstanceOf[String]) {
+ FileFactory.getUpdatedFilePath(tableLoc.get.asInstanceOf[String])
+ } else if (tableLoc.get.isInstanceOf[URI]) {
+ FileFactory.getUpdatedFilePath(tableLoc.get.asInstanceOf[URI].getPath)
+ }
+ case None =>
+ CarbonEnv.getTablePath(tableIdentifier.database, tableIdentifier.table)(sparkSession)
+ }
+ CarbonDatasourceHadoopRelation(sparkSession,
+ Array(tableLocation.asInstanceOf[String]),
+ catalogTable.storage.properties,
+ Option(catalogTable.schema))
+ case _ => throw new NoSuchTableException(tableIdentifier.database.get, tableIdentifier.table)
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 5a8e58b..cc0e6ab 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -16,9 +16,10 @@
*/
package org.apache.spark.sql.hive
-import org.apache.spark.sql.{RuntimeConfig, SparkSession}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, DataFrame, Dataset, RuntimeConfig, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.types.StructType
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
@@ -144,6 +145,31 @@ trait CarbonMetaStore {
def getThriftTableInfo(tablePath: CarbonTablePath)(sparkSession: SparkSession): TableInfo
def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable]
+
+ /**
+ * Method will be used to retrieve or create carbon data source relation
+ *
+ * @param sparkSession
+ * @param tableIdentifier
+ * @return
+ */
+ def createCarbonDataSourceHadoopRelation(
+ sparkSession: SparkSession,
+ tableIdentifier: TableIdentifier): CarbonDatasourceHadoopRelation
+
+ /**
+ * Method will be used retrieve the schema from unresolved relation
+ *
+ * @param sparkSession
+ * @param query
+ * @return
+ */
+ def getSchemaFromUnresolvedRelation(
+ sparkSession: SparkSession,
+ query: LogicalPlan): StructType = {
+ val df: DataFrame = Dataset.ofRows(sparkSession, query)
+ df.schema
+ }
}
/**
* Factory for Carbon metastore
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 79095ca..3597208 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -26,14 +26,12 @@ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkSqlAstBuilder
import org.apache.spark.sql.execution.command.{PartitionerField, TableModel, TableNewProcessor}
-import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
+import org.apache.spark.sql.execution.command.table.{CarbonCreateTableAsSelectCommand, CarbonCreateTableCommand}
import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.spark.CarbonOption
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -78,7 +76,9 @@ class CarbonSparkSqlParser(conf: SQLConf, sparkSession: SparkSession) extends Ab
}
}
-class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
+class CarbonHelperSqlAstBuilder(conf: SQLConf,
+ parser: CarbonSpark2SqlParser,
+ sparkSession: SparkSession)
extends SparkSqlAstBuilder(conf) {
def getFileStorage(createFileFormat: CreateFileFormatContext): String = {
@@ -147,7 +147,8 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
tablePropertyList : TablePropertyListContext,
locationSpecContext: SqlBaseParser.LocationSpecContext,
tableComment : Option[String],
- ctas: TerminalNode) : LogicalPlan = {
+ ctas: TerminalNode,
+ query: QueryContext) : LogicalPlan = {
// val parser = new CarbonSpark2SqlParser
val (tableIdentifier, temp, ifNotExists, external) = visitCreateTableHeader(tableHeader)
@@ -166,9 +167,6 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
if (external) {
operationNotAllowed("CREATE EXTERNAL TABLE", tableHeader)
}
- if (ctas != null && columns != null) {
- operationNotAllowed("CREATE TABLE AS SELECT", tableHeader)
- }
val cols = Option(columns).toSeq.flatMap(visitColTypeList)
val properties = getPropertyKeyValues(tablePropertyList)
@@ -210,13 +208,33 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
}
}
- val fields = parser.getFields(cols ++ partitionByStructFields)
val options = new CarbonOption(properties)
+ // validate streaming property
+ validateStreamingProperty(options)
+ var fields = parser.getFields(cols ++ partitionByStructFields)
+ // validate for create table as select
+ val selectQuery = Option(query).map(plan)
+ selectQuery match {
+ case Some(q) =>
+ // create table as select does not allow creation of partitioned table
+ if (partitionFields.nonEmpty) {
+ val errorMessage = "A Create Table As Select (CTAS) statement is not allowed to " +
+ "create a partitioned table using Carbondata file formats."
+ operationNotAllowed(errorMessage, partitionColumns)
+ }
+ // create table as select does not allow to explicitly specify schema
+ if (fields.nonEmpty) {
+ operationNotAllowed(
+ "Schema may not be specified in a Create Table As Select (CTAS) statement", columns)
+ }
+ fields = parser
+ .getFields(CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .getSchemaFromUnresolvedRelation(sparkSession, Some(q).get))
+ case _ =>
+ // ignore this case
+ }
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
-
- validateStreamingProperty(options)
-
// prepare table model of the collected tokens
val tableModel: TableModel = parser.prepareTableModel(
ifNotExists,
@@ -228,7 +246,19 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
bucketFields,
isAlterFlow = false,
tableComment)
- CarbonCreateTableCommand(TableNewProcessor(tableModel), tableModel.ifNotExistsSet, tablePath)
+
+ selectQuery match {
+ case query@Some(q) =>
+ CarbonCreateTableAsSelectCommand(
+ TableNewProcessor(tableModel),
+ query.get,
+ tableModel.ifNotExistsSet,
+ tablePath)
+ case _ =>
+ CarbonCreateTableCommand(TableNewProcessor(tableModel),
+ tableModel.ifNotExistsSet,
+ tablePath)
+ }
}
private def validateStreamingProperty(carbonOption: CarbonOption): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
index c198613..b55a6aa 100644
--- a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
@@ -259,25 +259,27 @@ object CarbonOptimizerUtil {
}
}
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser) extends
- SparkSqlAstBuilder(conf) {
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
- val helper = new CarbonHelperSqlAstBuilder(conf, parser)
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
val fileStorage = helper.getFileStorage(ctx.createFileFormat)
if (fileStorage.equalsIgnoreCase("'carbondata'") ||
fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(ctx.createTableHeader,
- ctx.skewSpec,
- ctx.bucketSpec,
- ctx.partitionColumns,
- ctx.columns,
- ctx.tablePropertyList,
- ctx.locationSpec,
- Option(ctx.STRING()).map(string),
- ctx.AS)
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
} else {
super.visitCreateTable(ctx)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/59de7cdb/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
index 8acddfa..c951e5e 100644
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
@@ -280,25 +280,27 @@ class CarbonOptimizer(
}
}
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser) extends
- SparkSqlAstBuilder(conf) {
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
- val helper = new CarbonHelperSqlAstBuilder(conf, parser)
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = {
val fileStorage = helper.getFileStorage(ctx.createFileFormat)
if (fileStorage.equalsIgnoreCase("'carbondata'") ||
fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(ctx.createTableHeader,
- ctx.skewSpec,
- ctx.bucketSpec,
- ctx.partitionColumns,
- ctx.columns,
- ctx.tablePropertyList,
- ctx.locationSpec(),
- Option(ctx.STRING()).map(string),
- ctx.AS)
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
} else {
super.visitCreateHiveTable(ctx)
}
[23/50] [abbrv] carbondata git commit: [CARBONDATA-1714] Fixed Issue
for Selection of null values after having multiple alter commands
Posted by ja...@apache.org.
[CARBONDATA-1714] Fixed Issue for Selection of null values after having multiple alter commands
In CarbonTableInputFormat, isFilterMesure is populated with measurelist size as this includes the count of previous drop column having invisible true.
This closes #1629
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/64b26513
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/64b26513
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/64b26513
Branch: refs/heads/fgdatamap
Commit: 64b26513f6ad7c75e48a26b867389d038196f872
Parents: 4c9af60
Author: Jatin <ja...@knoldus.in>
Authored: Thu Dec 7 10:41:37 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed Dec 20 20:46:16 2017 +0530
----------------------------------------------------------------------
.../hadoop/api/CarbonTableInputFormat.java | 3 ++-
.../AlterTableValidationTestCase.scala | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64b26513/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index e60a588..7db9c0a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -826,8 +826,9 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()];
+ // getAllMeasures returns list of visible and invisible columns
boolean[] isFilterMeasures =
- new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())];
+ new boolean[carbonTable.getAllMeasures().size()];
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, isFilterDimensions,
isFilterMeasures);
queryModel.setIsFilterDimensions(isFilterDimensions);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64b26513/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index 92d337f..e7e0160 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -527,6 +527,26 @@ class AlterTableValidationTestCase extends Spark2QueryTest with BeforeAndAfterAl
sql("drop table if exists preaggmain_new")
sql("drop table if exists preaggMain_preagg1")
}
+ test("test to check select columns after alter commands with null values"){
+ sql("drop table if exists restructure")
+ sql("drop table if exists restructure1")
+ sql(
+ "CREATE TABLE restructure (empno int, empname String, designation String, doj Timestamp, " +
+ "workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
+ "projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int," +
+ "utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE restructure OPTIONS
+ |('DELIMITER'= ',', 'QUOTECHAR'= '\"')""".stripMargin)
+ sql("ALTER TABLE restructure rename to restructure1")
+ sql("ALTER TABLE restructure1 ADD COLUMNS (projId int)")
+ sql("ALTER TABLE restructure1 DROP COLUMNS (projId)")
+ sql("ALTER TABLE restructure1 CHANGE empno empno BIGINT")
+ sql("ALTER TABLE restructure1 ADD COLUMNS (a1 INT, b1 STRING) TBLPROPERTIES('DICTIONARY_EXCLUDE'='b1')")
+ checkAnswer(sql("select a1,b1,empname from restructure1 where a1 is null and b1 is null and empname='arvind'"),Row(null,null,"arvind"))
+ sql("drop table if exists restructure1")
+ sql("drop table if exists restructure")
+ }
override def afterAll {
sql("DROP TABLE IF EXISTS restructure")
[03/50] [abbrv] carbondata git commit: [CARBONDATA-1898]Fixed Like,
Contains, Ends with query optimization With OR condition
Posted by ja...@apache.org.
[CARBONDATA-1898]Fixed Like, Contains, Ends with query optimization With OR condition
This closes #1667
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9a2b33eb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9a2b33eb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9a2b33eb
Branch: refs/heads/fgdatamap
Commit: 9a2b33ebd7acc7f8c934e73dcd5359d7c4e55f24
Parents: 8348752
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Dec 15 16:35:25 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Dec 16 21:33:30 2017 +0530
----------------------------------------------------------------------
.../apache/spark/sql/CarbonBoundReference.scala | 9 ---------
.../strategy/CarbonLateDecodeStrategy.scala | 5 -----
.../spark/sql/optimizer/CarbonFilters.scala | 18 ------------------
3 files changed, 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a2b33eb/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index 6f578ac..21e56b0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -48,12 +48,3 @@ case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nu
override def newInstance(): NamedExpression = throw new UnsupportedOperationException
}
-
-case class CarbonEndsWith(expr: Expression) extends Filter {
- override def references: Array[String] = null
-}
-
-case class CarbonContainsWith(expr: Expression) extends Filter {
- override def references: Array[String] = null
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a2b33eb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 79bbfb0..8b247f7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -471,7 +471,6 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
(unrecognizedPredicates ++ unhandledPredicates, translatedFilters)
}
-
/**
* Tries to translate a Catalyst [[Expression]] into data source [[Filter]].
*
@@ -568,10 +567,6 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
- case c@EndsWith(a: Attribute, Literal(v, t)) =>
- Some(CarbonEndsWith(c))
- case c@Contains(a: Attribute, Literal(v, t)) =>
- Some(CarbonContainsWith(c))
case others => None
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a2b33eb/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 75cc128..7408594 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -24,8 +24,6 @@ import org.apache.spark.sql.CastExpr
import org.apache.spark.sql.SparkUnknownExpression
import org.apache.spark.sql.sources
import org.apache.spark.sql.types._
-import org.apache.spark.sql.CarbonContainsWith
-import org.apache.spark.sql.CarbonEndsWith
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
@@ -108,18 +106,6 @@ object CarbonFilters {
val r = new LessThanExpression(
getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit))
Some(new AndExpression(l, r))
- case CarbonEndsWith(expr: Expression) =>
- Some(new SparkUnknownExpression(expr.transform {
- case AttributeReference(name, dataType, _, _) =>
- CarbonBoundReference(new CarbonColumnExpression(name.toString,
- CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
- }))
- case CarbonContainsWith(expr: Expression) =>
- Some(new SparkUnknownExpression(expr.transform {
- case AttributeReference(name, dataType, _, _) =>
- CarbonBoundReference(new CarbonColumnExpression(name.toString,
- CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
- }))
case CastExpr(expr: Expression) =>
Some(transformExpression(expr))
case _ => None
@@ -250,10 +236,6 @@ object CarbonFilters {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
- case c@EndsWith(a: Attribute, Literal(v, t)) =>
- Some(CarbonEndsWith(c))
- case c@Contains(a: Attribute, Literal(v, t)) =>
- Some(CarbonContainsWith(c))
case c@Cast(a: Attribute, _) =>
Some(CastExpr(c))
case others =>
[30/50] [abbrv] carbondata git commit: [CARBONDATA-1439] Wrong Error
message shown for Bad records even when BAD_RECORDS_LOGGER_ENABLE is set to
true
Posted by ja...@apache.org.
[CARBONDATA-1439] Wrong Error message shown for Bad records even when BAD_RECORDS_LOGGER_ENABLE is set to true
Correct the error message for bad records scenario when BAD_RECORDS_LOGGER_ENABLE is set to true.
This closes #1311
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0c0f90ca
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0c0f90ca
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0c0f90ca
Branch: refs/heads/fgdatamap
Commit: 0c0f90ca7f8353c38e5641dee476456286c36b76
Parents: 425f820
Author: shivangi <sh...@knoldus.in>
Authored: Fri Aug 4 13:25:02 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Dec 21 14:15:07 2017 +0530
----------------------------------------------------------------------
.../processing/loading/converter/impl/RowConverterImpl.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0c0f90ca/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
index 83245a3..959e4f5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
@@ -165,8 +165,10 @@ public class RowConverterImpl implements RowConverter {
if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) {
badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
if (badRecordLogger.isDataLoadFail()) {
- String error = "Data load failed due to bad record: " + logHolder.getReason() +
- "Please enable bad record logger to know the detail reason.";
+ String error = "Data load failed due to bad record: " + logHolder.getReason();
+ if (!badRecordLogger.isBadRecordLoggerEnable()) {
+ error += "Please enable bad record logger to know the detail reason.";
+ }
throw new BadRecordFoundException(error);
}
logHolder.clear();
[31/50] [abbrv] carbondata git commit: [CARBONDATA-1698]Adding
support for table level compaction configuration
Posted by ja...@apache.org.
[CARBONDATA-1698]Adding support for table level compaction configuration
Adding support for table level compaction configuration
This closes #1575
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f1c6ddde
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f1c6ddde
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f1c6ddde
Branch: refs/heads/fgdatamap
Commit: f1c6dddec8d7057dfc1a1f70555faec4c6184a52
Parents: 0c0f90c
Author: Jin Zhou <xa...@yeah.net>
Authored: Fri Dec 15 19:53:41 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Thu Dec 21 17:07:44 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 14 +-
.../carbondata/core/util/CarbonProperties.java | 10 +-
docs/data-management-on-carbondata.md | 27 +-
.../TableLevelCompactionOptionExample.scala | 106 +++++++
.../TestCreateTableWithCompactionOptions.scala | 198 ++++++++++++++
...CompactionSupportGlobalSortBigFileTest.scala | 4 +-
.../TableLevelCompactionOptionTest.scala | 274 +++++++++++++++++++
.../carbondata/spark/util/CommonUtil.scala | 119 ++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +
.../spark/rdd/CarbonDataRDDFactory.scala | 7 +-
.../CarbonAlterTableCompactionCommand.scala | 3 +-
.../table/CarbonDescribeFormattedCommand.scala | 32 +++
.../processing/merger/CarbonDataMergerUtil.java | 107 +++++---
13 files changed, 854 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 9534099..71ab668 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -789,12 +789,12 @@ public final class CarbonCommonConstants {
* Size of Major Compaction in MBs
*/
@CarbonProperty
- public static final String MAJOR_COMPACTION_SIZE = "carbon.major.compaction.size";
+ public static final String CARBON_MAJOR_COMPACTION_SIZE = "carbon.major.compaction.size";
/**
* By default size of major compaction in MBs.
*/
- public static final String DEFAULT_MAJOR_COMPACTION_SIZE = "1024";
+ public static final String DEFAULT_CARBON_MAJOR_COMPACTION_SIZE = "1024";
/**
* This property is used to tell how many segments to be preserved from merging.
@@ -873,6 +873,16 @@ public final class CarbonCommonConstants {
public static final String TABLE_BLOCKSIZE = "table_blocksize";
// set in column level to disable inverted index
public static final String NO_INVERTED_INDEX = "no_inverted_index";
+ // table property name of major compaction size
+ public static final String TABLE_MAJOR_COMPACTION_SIZE = "major_compaction_size";
+ // table property name of auto load merge
+ public static final String TABLE_AUTO_LOAD_MERGE = "auto_load_merge";
+ // table property name of compaction level threshold
+ public static final String TABLE_COMPACTION_LEVEL_THRESHOLD = "compaction_level_threshold";
+ // table property name of preserve segments numbers while compaction
+ public static final String TABLE_COMPACTION_PRESERVE_SEGMENTS = "compaction_preserve_segments";
+ // table property name of allowed compaction days while compaction
+ public static final String TABLE_ALLOWED_COMPACTION_DAYS = "allowed_compaction_days";
/**
* 16 mb size
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 11aea99..7b80a8b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -620,10 +620,12 @@ public final class CarbonProperties {
public long getMajorCompactionSize() {
long compactionSize;
try {
- compactionSize = Long.parseLong(getProperty(CarbonCommonConstants.MAJOR_COMPACTION_SIZE,
- CarbonCommonConstants.DEFAULT_MAJOR_COMPACTION_SIZE));
+ compactionSize = Long.parseLong(getProperty(
+ CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE,
+ CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE));
} catch (NumberFormatException e) {
- compactionSize = Long.parseLong(CarbonCommonConstants.DEFAULT_MAJOR_COMPACTION_SIZE);
+ compactionSize = Long.parseLong(
+ CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE);
}
return compactionSize;
}
@@ -684,7 +686,7 @@ public final class CarbonProperties {
* @param commaSeparatedLevels the string format value before separating
* @return the int array format value after separating by comma
*/
- private int[] getIntArray(String commaSeparatedLevels) {
+ public int[] getIntArray(String commaSeparatedLevels) {
String[] levels = commaSeparatedLevels.split(",");
int[] compactionSize = new int[levels.length];
int i = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 7f6e3b9..94820ad 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -92,6 +92,26 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
```
NOTE: 512 or 512M both are accepted.
+ - **Table Compaction Configuration**
+
+ These properties are table level compaction configurations, if not specified, system level configurations in carbon.properties will be used.
+ Following are 5 configurations:
+
+ * MAJOR_COMPACTION_SIZE: same meaning with carbon.major.compaction.size, size in MB.
+ * AUTO_LOAD_MERGE: same meaning with carbon.enable.auto.load.merge.
+ * COMPACTION_LEVEL_THRESHOLD: same meaning with carbon.compaction.level.threshold.
+ * COMPACTION_PRESERVE_SEGMENTS: same meaning with carbon.numberof.preserve.segments.
+ * ALLOWED_COMPACTION_DAYS: same meaning with carbon.allowed.compaction.days.
+
+ ```
+ TBLPROPERTIES ('MAJOR_COMPACTION_SIZE'='2048',
+ 'AUTO_LOAD_MERGE'='true',
+ 'COMPACTION_LEVEL_THRESHOLD'='5,6',
+ 'COMPACTION_PRESERVE_SEGMENTS'='10',
+ 'ALLOWED_COMPACTION_DAYS'='5')
+ ```
+
+
### Example:
```
@@ -109,7 +129,12 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
'NO_INVERTED_INDEX'='productBatch',
'SORT_COLUMNS'='productName,storeCity',
'SORT_SCOPE'='NO_SORT',
- 'TABLE_BLOCKSIZE'='512')
+ 'TABLE_BLOCKSIZE'='512',
+ 'MAJOR_COMPACTION_SIZE'='2048',
+ 'AUTO_LOAD_MERGE'='true',
+ 'COMPACTION_LEVEL_THRESHOLD'='5,6',
+ 'COMPACTION_PRESERVE_SEGMENTS'='10',
+ 'ALLOWED_COMPACTION_DAYS'='5')
```
## TABLE MANAGEMENT
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/examples/spark2/src/main/scala/org/apache/carbondata/examples/TableLevelCompactionOptionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/TableLevelCompactionOptionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/TableLevelCompactionOptionExample.scala
new file mode 100644
index 0000000..8b34a8b
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/TableLevelCompactionOptionExample.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+object TableLevelCompactionOptionExample {
+
+ def main(args: Array[String]) {
+ val spark = ExampleUtils.createCarbonSession("DataManagementExample")
+ spark.sparkContext.setLogLevel("WARN")
+
+ spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ // Create table with table level compaction options
+ // while loading and compacting, table level compaction options will be used instead of
+ // options specified in carbon.properties
+ spark.sql(
+ s"""
+ | CREATE TABLE IF NOT EXISTS carbon_table(
+ | ID Int,
+ | date Date,
+ | country String,
+ | name String,
+ | phonetype String,
+ | serialname String,
+ | salary Int,
+ | floatField float
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES (
+ | 'MAJOR_COMPACTION_SIZE'='1024',
+ | 'AUTO_LOAD_MERGE'='true',
+ | 'COMPACTION_LEVEL_THRESHOLD'='3,2',
+ | 'COMPACTION_PRESERVE_SEGMENTS'='2',
+ | 'ALLOWED_COMPACTION_DAYS'='1')
+ """.stripMargin)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val path = s"$rootPath/examples/spark2/src/main/resources/dataSample.csv"
+
+ // load 6 segments
+ // scalastyle:off
+ (1 to 6).foreach(_ => spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin))
+ // scalastyle:on
+
+ // show all segments, existing segments are 0.1,3,4,5, compacted segments are 0,1,2
+ // because of 2 segments are preserved, only one level-1 minor compaction is triggered
+ spark.sql("SHOW SEGMENTS FOR TABLE carbon_table").show()
+
+ // load another 2 segments
+ // scalastyle:off
+ (1 to 2).foreach(_ => spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin))
+ // scalastyle:on
+
+ // show all segments, existing segments will be 0.2,6,7,
+ // compacted segments are 0,1,2,3,4,5,0.1,3.1
+ spark.sql("SHOW SEGMENTS FOR TABLE carbon_table").show()
+
+ // load another 2 segments
+ // scalastyle:off
+ (1 to 2).foreach(_ => spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin))
+ // scalastyle:on
+
+ // do major compaction, there will be 3 segment left(2 preserved segments)
+ spark.sql("ALTER TABLE carbon_table COMPACT 'MAJOR'")
+ spark.sql("CLEAN FILES FOR TABLE carbon_table")
+ spark.sql("SHOW SEGMENTS FOR TABLE carbon_table").show()
+
+ // Drop table
+ spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ spark.stop()
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithCompactionOptions.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithCompactionOptions.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithCompactionOptions.scala
new file mode 100644
index 0000000..d7c05ce
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithCompactionOptions.scala
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the"License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an"AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateTableWithCompactionOptions extends QueryTest with BeforeAndAfterAll {
+
+ val tableWithCompactionOptions = "tableWithCompactionOptions"
+ val tableWithInvalidMajorCompactionSize = "tableWithInvalidMajorCompactionSize"
+ val tableWithInvalidAutoLoadMerge = "tableWithInvalidAutoLoadMerge"
+ val tableWithInvalidLevelThreshold = "tableWithInvalidLevelThreshold"
+ val tableWithInvalidPreserveSegments = "tableWithInvalidPreserveSegments"
+ val tableWithInvalidAllowedDays = "tableWithInvalidAllowedDays"
+ val tableWithoutCompactionOptions = "tableWithoutCompactionOptions"
+
+ override def beforeAll: Unit = {
+ cleanTables()
+ }
+
+ override def afterAll: Unit = {
+ cleanTables()
+ }
+
+ private def cleanTables(): Unit = {
+ sql("use default")
+ sql(s"DROP TABLE IF EXISTS $tableWithCompactionOptions")
+ sql(s"DROP TABLE IF EXISTS $tableWithInvalidMajorCompactionSize")
+ sql(s"DROP TABLE IF EXISTS $tableWithInvalidAutoLoadMerge")
+ sql(s"DROP TABLE IF EXISTS $tableWithInvalidLevelThreshold")
+ sql(s"DROP TABLE IF EXISTS $tableWithInvalidPreserveSegments")
+ sql(s"DROP TABLE IF EXISTS $tableWithInvalidAllowedDays")
+ sql(s"DROP TABLE IF EXISTS $tableWithoutCompactionOptions")
+ }
+
+ test("test create table with compaction options") {
+ sql(
+ s"""
+ | CREATE TABLE $tableWithCompactionOptions(
+ | intField INT,
+ | stringField STRING
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('MAJOR_COMPACTION_SIZE'='10240',
+ | 'AUTO_LOAD_MERGE'='true',
+ | 'COMPACTION_LEVEL_THRESHOLD'='5,6',
+ | 'COMPACTION_PRESERVE_SEGMENTS'='10',
+ | 'ALLOWED_COMPACTION_DAYS'='5')
+ """.stripMargin)
+
+ val tableOptions = sql(s"DESCRIBE FORMATTED $tableWithCompactionOptions")
+ .collect().map(r => (r.getString(0).trim, r.getString(1).trim)).toMap
+
+ assert(tableOptions.contains("MAJOR_COMPACTION_SIZE"))
+ assert(tableOptions.getOrElse("MAJOR_COMPACTION_SIZE","").equals("10240"))
+ assert(tableOptions.contains("AUTO_LOAD_MERGE"))
+ assert(tableOptions.getOrElse("AUTO_LOAD_MERGE","").equals("true"))
+ assert(tableOptions.contains("COMPACTION_LEVEL_THRESHOLD"))
+ assert(tableOptions.getOrElse("COMPACTION_LEVEL_THRESHOLD","").equals("5,6"))
+ assert(tableOptions.contains("COMPACTION_PRESERVE_SEGMENTS"))
+ assert(tableOptions.getOrElse("COMPACTION_PRESERVE_SEGMENTS","").equals("10"))
+ assert(tableOptions.contains("ALLOWED_COMPACTION_DAYS"))
+ assert(tableOptions.getOrElse("ALLOWED_COMPACTION_DAYS","").equals("5"))
+ }
+
+ test("test create table with invalid major compaction size") {
+ val exception: Exception = intercept[Exception] {
+ sql(
+ s"""
+ |CREATE TABLE $tableWithInvalidMajorCompactionSize
+ |(
+ |intField INT,
+ |stringField STRING
+ |)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES('MAJOR_COMPACTION_SIZE'='abc')
+ """.stripMargin)
+ }
+ assert(exception.getMessage.contains(
+ "Invalid major_compaction_size value found: abc, " +
+ "only int value greater than 0 is supported."))
+ }
+
+ test("test create table with invalid auto load merge") {
+ val exception: Exception = intercept[Exception] {
+ sql(
+ s"""
+ |CREATE TABLE $tableWithInvalidAutoLoadMerge
+ |(
+ |intField INT,
+ |stringField STRING
+ |)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES('AUTO_LOAD_MERGE'='123')
+ """.stripMargin)
+ }
+ assert(exception.getMessage.contains(
+ "Invalid auto_load_merge value found: 123, only true|false is supported."))
+ }
+
+ test("test create table with invalid level threshold") {
+ val exception: Exception = intercept[Exception] {
+ sql(
+ s"""
+ |CREATE TABLE $tableWithInvalidLevelThreshold
+ |(
+ |intField INT,
+ |stringField STRING
+ |)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES(
+ |'AUTO_LOAD_MERGE'='true',
+ |'COMPACTION_LEVEL_THRESHOLD'='x,6')
+ """.stripMargin)
+ }
+ assert(exception.getMessage.contains(
+ "Invalid compaction_level_threshold value found: x,6, " +
+ "only int values separated by comma and between 0 and 100 are supported."))
+ }
+
+ test("test create table with invalid preserve segments number") {
+ val exception: Exception = intercept[Exception] {
+ sql(
+ s"""
+ |CREATE TABLE $tableWithInvalidPreserveSegments
+ |(
+ |intField INT,
+ |stringField STRING
+ |)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES(
+ |'AUTO_LOAD_MERGE'='true',
+ |'COMPACTION_LEVEL_THRESHOLD'='4,6',
+ |'COMPACTION_PRESERVE_SEGMENTS'='abc')
+ """.stripMargin)
+ }
+ assert(exception.getMessage.contains(
+ "Invalid compaction_preserve_segments value found: abc, " +
+ "only int value between 0 and 100 is supported."))
+ }
+
+ test("test create table with invalid allowed days") {
+ val exception: Exception = intercept[Exception] {
+ sql(
+ s"""
+ |CREATE TABLE $tableWithInvalidAllowedDays
+ |(
+ |intField INT,
+ |stringField STRING
+ |)
+ |STORED BY 'carbondata'
+ |TBLPROPERTIES(
+ |'AUTO_LOAD_MERGE'='true',
+ |'ALLOWED_COMPACTION_DAYS'='abc')
+ """.stripMargin)
+ }
+ assert(exception.getMessage.contains(
+ "Invalid allowed_compaction_days value found: abc, " +
+ "only int value between 0 and 100 is supported."))
+ }
+
+ test("test create table without compaction options") {
+ sql(
+ s"""
+ | CREATE TABLE $tableWithoutCompactionOptions(
+ | intField INT,
+ | stringField STRING
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val tableOptions = sql(s"DESCRIBE FORMATTED $tableWithoutCompactionOptions")
+ .collect().map(r => (r.getString(0).trim, r.getString(1).trim)).toMap
+
+ assert(!tableOptions.contains("MAJOR_COMPACTION_SIZE"))
+ assert(!tableOptions.contains("AUTO_LOAD_MERGE"))
+ assert(!tableOptions.contains("COMPACTION_LEVEL_THRESHOLD"))
+ assert(!tableOptions.contains("COMPACTION_PRESERVE_SEGMENTS"))
+ assert(!tableOptions.contains("ALLOWED_COMPACTION_DAYS"))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
index 6d79f6c..c522c1e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
@@ -51,7 +51,7 @@ class CompactionSupportGlobalSortBigFileTest extends QueryTest with BeforeAndAft
CompactionSupportGlobalSortBigFileTest.deleteFile(file3)
CompactionSupportGlobalSortBigFileTest.deleteFile(file4)
CompactionSupportGlobalSortBigFileTest.deleteFile(file5)
- resetConf(CarbonCommonConstants.DEFAULT_MAJOR_COMPACTION_SIZE)
+ resetConf(CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE)
}
override def beforeEach {
@@ -104,7 +104,7 @@ class CompactionSupportGlobalSortBigFileTest extends QueryTest with BeforeAndAft
private def resetConf(size:String) {
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.MAJOR_COMPACTION_SIZE, size)
+ .addProperty(CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE, size)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala
new file mode 100644
index 0000000..5445779
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/TableLevelCompactionOptionTest.scala
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the"License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an"AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.datacompaction
+
+import java.io.{File, PrintWriter}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+class TableLevelCompactionOptionTest extends QueryTest
+ with BeforeAndAfterEach with BeforeAndAfterAll {
+
+ val tempFilePath: String = s"$resourcesPath/temp/tableLevelCompactionParaTest.csv"
+ val sampleFilePath: String = resourcesPath + "/sample.csv"
+
+ override def beforeEach {
+ cleanTable()
+ }
+
+ override def afterEach {
+ resetConf()
+ cleanTable()
+ }
+
+ private def resetConf() ={
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE,
+ CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
+ CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER,
+ CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
+ CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT)
+ }
+
+ private def cleanTable() = {
+ deleteTempFile()
+ sql("DROP TABLE IF EXISTS carbon_table")
+ }
+
+ private def generateTempFile() = {
+ val writer = new PrintWriter(new File(tempFilePath))
+ try {
+ writer.println("id,name,city,age")
+ val lines =
+ s"""|1,david,shenzhen,31
+ |2,eason,shenzhen,27
+ |3,jarry,wuhan,35
+ |3,jarry,Bangalore,35
+ |4,kunal,Delhi,26
+ |4,vishal,Bangalore,29""".stripMargin
+ for (i <- 0 until 250000) {
+ writer.println(lines)
+ }
+ writer.flush()
+ } finally {
+ if (writer != null) writer.close()
+ }
+ }
+
+ private def deleteTempFile() = {
+ val file = new File(tempFilePath)
+ if (file.exists()) {
+ file.delete()
+ }
+ }
+
+ test("MAJOR_COMPACTION_SIZE, use system level configuration"){
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE, "10")
+
+ // generate a temp file which is larger than 1M but smaller than 5M
+ generateTempFile()
+
+ sql(
+ """
+ |CREATE TABLE carbon_table
+ |(id INT, name STRING, city STRING, age INT)
+ |STORED BY 'org.apache.carbondata.format'
+ |TBLPROPERTIES('SORT_COLUMNS'='city,name')
+ """.stripMargin)
+
+ for (i <- 0 until 2) {
+ sql(s"LOAD DATA LOCAL INPATH '$tempFilePath' INTO TABLE carbon_table")
+ }
+
+ sql("ALTER TABLE carbon_table COMPACT 'MAJOR'")
+ sql("CLEAN FILES FOR TABLE carbon_table")
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(!SegmentSequenceIds.contains("0"))
+ assert(!SegmentSequenceIds.contains("1"))
+ assert(SegmentSequenceIds.contains("0.1"))
+
+ }
+
+ test("MAJOR_COMPACTION_SIZE, use table level configuration") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE, "10")
+
+ // generate a temp file which is larger than 1M but smaller than 5M
+ generateTempFile()
+
+ sql(
+ """
+ |CREATE TABLE carbon_table
+ |(id INT, name STRING, city STRING, age INT)
+ |STORED BY 'org.apache.carbondata.format'
+ |TBLPROPERTIES('SORT_COLUMNS'='city,name',
+ |'MAJOR_COMPACTION_SIZE'='1')
+ """.stripMargin)
+
+ for (i <- 0 until 2) {
+ sql(s"LOAD DATA LOCAL INPATH '$tempFilePath' INTO TABLE carbon_table")
+ }
+
+ // each segment is larger than 1M, so no segments will be compacted
+ sql("ALTER TABLE carbon_table COMPACT 'MAJOR'")
+ sql("CLEAN FILES FOR TABLE carbon_table")
+
+ val segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ val SegmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(SegmentSequenceIds.contains("0"))
+ assert(SegmentSequenceIds.contains("1"))
+ assert(!SegmentSequenceIds.contains("0.1"))
+
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE,
+ CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE)
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: true, use system level configuration"){
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "4,2")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER, "0")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT, "0")
+
+ sql(
+ """
+ | CREATE TABLE carbon_table(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name')
+ """.stripMargin)
+
+ for (i <- 0 until 8) {
+ sql(s"LOAD DATA LOCAL INPATH '$sampleFilePath' INTO TABLE carbon_table")
+ }
+ sql("CLEAN FILES FOR TABLE carbon_table")
+ var segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ var segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(segmentSequenceIds.size==1)
+ assert(segmentSequenceIds.contains("0.2"))
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: false, use table level configuration"){
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "4,2")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER, "0")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT, "0")
+
+ sql(
+ """
+ | CREATE TABLE carbon_table(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name',
+ | 'AUTO_LOAD_MERGE'='false')
+ """.stripMargin)
+
+ for (i <- 0 until 8) {
+ sql(s"LOAD DATA LOCAL INPATH '$sampleFilePath' INTO TABLE carbon_table")
+ }
+ // table level configuration: 'AUTO_LOAD_MERGE'='false', so no segments will be compacted
+ checkExistence(sql("SHOW SEGMENTS FOR TABLE carbon_table"), false, "Compacted")
+ var segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ var segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(segmentSequenceIds.size==8)
+ assert(!segmentSequenceIds.contains("0.1"))
+ assert(!segmentSequenceIds.contains("4.1"))
+ assert(!segmentSequenceIds.contains("0.2"))
+ }
+
+ test("ENABLE_AUTO_LOAD_MERGE: true, use table level configuration") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "4,2")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER, "0")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT, "0")
+
+ sql(
+ """
+ | CREATE TABLE carbon_table(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name',
+ | 'AUTO_LOAD_MERGE'='true',
+ | 'COMPACTION_LEVEL_THRESHOLD'='3,2',
+ | 'COMPACTION_PRESERVE_SEGMENTS'='2',
+ | 'TABLE_ALLOWED_COMPACTION_DAYS'='1')
+ """.stripMargin)
+
+ // load 6 segments, the latest 2 segments are preserved
+ // only one level-1 minor compaction is triggered which compacts segment 0,1,2 to segment 0.1
+ // seg0 \
+ // seg1 -- compacted to seg0.1
+ // seg2 /
+ // seg3
+ // seg4 (preserved)
+ // seg5 (preserved)
+ for (i <- 0 until 6) {
+ sql(s"LOAD DATA LOCAL INPATH '$sampleFilePath' INTO TABLE carbon_table")
+ }
+ sql("CLEAN FILES FOR TABLE carbon_table")
+ var segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ var segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(segmentSequenceIds.contains("0.1"))
+ assert(!segmentSequenceIds.contains("3.1"))
+ assert(!segmentSequenceIds.contains("0.2"))
+
+ // load another two segments, the latest 2 segments are preserved
+ // level-2 minor compaction is triggered which compacts segment 0,1,2,3,4,5 -> 0.2
+ // seg0 \
+ // seg1 -- compacted to seg0.1 \
+ // seg2 / -- compacted to seg0.2
+ // seg3 \ /
+ // seg4 -- compacted to seg3.1
+ // seg5 /
+ // seg6 (preserved)
+ // seg7 (preserved)
+ for (i <- 0 until 2) {
+ sql(s"LOAD DATA LOCAL INPATH '$sampleFilePath' INTO TABLE carbon_table")
+ }
+ sql("CLEAN FILES FOR TABLE carbon_table")
+ segments = sql("SHOW SEGMENTS FOR TABLE carbon_table")
+ segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ assert(segmentSequenceIds.contains("0.2"))
+ assert(!segmentSequenceIds.contains("0.1"))
+ assert(!segmentSequenceIds.contains("3.1"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index c902537..ae30300 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -471,6 +471,125 @@ object CommonUtil {
}
/**
+ * validate table level properties for compaction
+ *
+ * @param tableProperties
+ */
+ def validateTableLevelCompactionProperties(tableProperties: Map[String, String]): Unit = {
+ validateMajorCompactionSize(tableProperties)
+ validateAutoLoadMerge(tableProperties)
+ validateCompactionLevelThreshold(tableProperties)
+ validateCompactionPreserveSegmentsOrAllowedDays(tableProperties,
+ CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS)
+ validateCompactionPreserveSegmentsOrAllowedDays(tableProperties,
+ CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS)
+ }
+
+ /**
+ * This method will validate the major compaction size specified by the user
+ * the property is used while doing major compaction
+ *
+ * @param tableProperties
+ */
+ def validateMajorCompactionSize(tableProperties: Map[String, String]): Unit = {
+ var majorCompactionSize: Integer = 0
+ val tblPropName = CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE
+ if (tableProperties.get(tblPropName).isDefined) {
+ val majorCompactionSizeStr: String =
+ parsePropertyValueStringInMB(tableProperties(tblPropName))
+ try {
+ majorCompactionSize = Integer.parseInt(majorCompactionSizeStr)
+ } catch {
+ case e: NumberFormatException =>
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$majorCompactionSizeStr, only int value greater than 0 is supported.")
+ }
+ if (majorCompactionSize < 0) {
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$majorCompactionSizeStr, only int value greater than 0 is supported.")
+ }
+ tableProperties.put(tblPropName, majorCompactionSizeStr)
+ }
+ }
+
+ /**
+ * This method will validate the auto merge load property specified by the user
+ * the property is used while doing minor compaction
+ *
+ * @param tableProperties
+ */
+ def validateAutoLoadMerge(tableProperties: Map[String, String]): Unit = {
+ val tblPropName = CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE
+ if (tableProperties.get(tblPropName).isDefined) {
+ val trimStr = tableProperties(tblPropName).trim
+ if (!trimStr.equalsIgnoreCase("true") && !trimStr.equalsIgnoreCase("false")) {
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$trimStr, only true|false is supported.")
+ }
+ tableProperties.put(tblPropName, trimStr)
+ }
+ }
+
+ /**
+ * This method will validate the compaction level threshold property specified by the user
+ * the property is used while doing minor compaction
+ *
+ * @param tableProperties
+ */
+ def validateCompactionLevelThreshold(tableProperties: Map[String, String]): Unit = {
+ val tblPropName = CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD
+ if (tableProperties.get(tblPropName).isDefined) {
+ val regularedStr = tableProperties(tblPropName).replace(" ", "")
+ try {
+ val levels: Array[String] = regularedStr.split(",")
+ val thresholds = regularedStr.split(",").map(levelThresholdStr => levelThresholdStr.toInt)
+ if (!thresholds.forall(t => t < 100 && t > 0)) {
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$regularedStr, only int values separated by comma and between 0 " +
+ s"and 100 are supported.")
+ }
+ }
+ catch {
+ case e: NumberFormatException =>
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$regularedStr, only int values separated by comma and between 0 " +
+ s"and 100 are supported.")
+ }
+ tableProperties.put(tblPropName, regularedStr)
+ }
+ }
+
+ /**
+ * This method will validate the compaction preserve segments property
+ * or compaction allowed days property
+ *
+ * @param tableProperties
+ * @param tblPropName
+ */
+ def validateCompactionPreserveSegmentsOrAllowedDays(tableProperties: Map[String, String],
+ tblPropName: String): Unit = {
+ if (tblPropName.equals(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS) ||
+ tblPropName.equals(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS)) {
+ var propValue: Integer = 0
+ if (tableProperties.get(tblPropName).isDefined) {
+ val propValueStr = tableProperties(tblPropName).trim
+ try {
+ propValue = Integer.parseInt(propValueStr)
+ } catch {
+ case e: NumberFormatException =>
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$propValueStr, only int value between 0 and 100 is supported.")
+ }
+ if (propValue < 0 || propValue > 100) {
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$propValueStr, only int value between 0 and 100 is supported.")
+ }
+ tableProperties.put(tblPropName, propValueStr)
+ }
+ }
+ }
+
+ /**
* This method will validate the table block size specified by the user
*
* @param tableProperties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index a39465f..6e9b36c 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -275,6 +275,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// validate the tableBlockSize from table properties
CommonUtil.validateTableBlockSize(tableProperties)
+ // validate table level properties for compaction
+ CommonUtil.validateTableLevelCompactionProperties(tableProperties)
TableModel(
ifNotExistPresent,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8c9700c..d288122 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -208,7 +208,8 @@ object CarbonDataRDDFactory {
val newCarbonLoadModel = prepareCarbonLoadModel(table)
- val compactionSize = CarbonDataMergerUtil.getCompactionSize(CompactionType.MAJOR)
+ val compactionSize = CarbonDataMergerUtil
+ .getCompactionSize(CompactionType.MAJOR, carbonLoadModel)
val newcompactionModel = CompactionModel(
compactionSize,
@@ -720,8 +721,8 @@ object CarbonDataRDDFactory {
operationContext: OperationContext
): Unit = {
LOGGER.info(s"compaction need status is" +
- s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired() }")
- if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired()) {
+ s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired(carbonTable) }")
+ if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired(carbonTable)) {
LOGGER.audit(s"Compaction request received for table " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
val compactionSize = 0
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index cb0ff2d..517fbda 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -121,7 +121,8 @@ case class CarbonAlterTableCompactionCommand(
operationContext: OperationContext): Unit = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getName)
val compactionType = CompactionType.valueOf(alterTableModel.compactionType.toUpperCase)
- val compactionSize: Long = CarbonDataMergerUtil.getCompactionSize(compactionType)
+ val compactionSize: Long = CarbonDataMergerUtil
+ .getCompactionSize(compactionType, carbonLoadModel)
if (CompactionType.IUD_UPDDEL_DELTA == compactionType) {
if (alterTableModel.segmentUpdateStatusManager.isDefined) {
carbonLoadModel.setSegmentUpdateStatusManager(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 807c8e5..ccdbd6e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -107,6 +107,38 @@ private[sql] case class CarbonDescribeFormattedCommand(
val isStreaming = carbonTable.getTableInfo.getFactTable.getTableProperties.asScala
.getOrElse("streaming", "false")
results ++= Seq(("Streaming", isStreaming, ""))
+
+ val tblProps = carbonTable.getTableInfo.getFactTable.getTableProperties
+ results ++= Seq(("SORT_SCOPE", tblProps.getOrDefault("sort_scope", CarbonCommonConstants
+ .LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+
+ // show table level compaction options
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE)) {
+ results ++= Seq((CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE.toUpperCase
+ , tblProps.get(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE),
+ CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE))
+ }
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE)) {
+ results ++= Seq((CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE.toUpperCase,
+ tblProps.get(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE),
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE))
+ }
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD)) {
+ results ++= Seq((CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD.toUpperCase,
+ tblProps.get(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD),
+ CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD))
+ }
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS)) {
+ results ++= Seq((CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS.toUpperCase,
+ tblProps.get(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS),
+ CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER))
+ }
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS)) {
+ results ++= Seq((CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS.toUpperCase,
+ tblProps.get(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS),
+ CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT))
+ }
+
results ++= Seq(("", "", ""), ("##Detailed Column property", "", ""))
if (colPropStr.length() > 0) {
results ++= Seq((colPropStr, "", ""))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1c6ddde/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 3729b1d..e245927 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -21,15 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -78,7 +70,8 @@ public final class CarbonDataMergerUtil {
// carbon data file case.
CarbonFile[] factFile = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
+ @Override
+ public boolean accept(CarbonFile file) {
return CarbonTablePath.isCarbonDataFile(file.getName());
}
});
@@ -93,17 +86,22 @@ public final class CarbonDataMergerUtil {
/**
* To check whether the merge property is enabled or not.
*
+ * @Params carbonTable
* @return
*/
-
- public static boolean checkIfAutoLoadMergingRequired() {
+ public static boolean checkIfAutoLoadMergingRequired(CarbonTable carbonTable) {
// load merge is not supported as per new store format
// moving the load merge check in early to avoid unnecessary load listing causing IOException
// check whether carbons segment merging operation is enabled or not.
// default will be false.
+ Map<String, String> tblProps = carbonTable.getTableInfo().getFactTable().getTableProperties();
+
String isLoadMergeEnabled = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
- CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE);
+ .getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE);
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE)) {
+ isLoadMergeEnabled = tblProps.get(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE);
+ }
if (isLoadMergeEnabled.equalsIgnoreCase("false")) {
return false;
}
@@ -386,9 +384,11 @@ public final class CarbonDataMergerUtil {
* @return
*/
public static List<LoadMetadataDetails> identifySegmentsToBeMerged(
- CarbonLoadModel carbonLoadModel, long compactionSize,
- List<LoadMetadataDetails> segments, CompactionType compactionType) {
+ CarbonLoadModel carbonLoadModel, long compactionSize,
+ List<LoadMetadataDetails> segments, CompactionType compactionType) {
String tablePath = carbonLoadModel.getTablePath();
+ Map<String, String> tableLevelProperties = carbonLoadModel.getCarbonDataLoadSchema()
+ .getCarbonTable().getTableInfo().getFactTable().getTableProperties();
List<LoadMetadataDetails> sortedSegments = new ArrayList<LoadMetadataDetails>(segments);
sortSegments(sortedSegments);
@@ -402,22 +402,24 @@ public final class CarbonDataMergerUtil {
// check preserve property and preserve the configured number of latest loads.
List<LoadMetadataDetails> listOfSegmentsAfterPreserve =
- checkPreserveSegmentsPropertyReturnRemaining(sortedSegments);
+ checkPreserveSegmentsPropertyReturnRemaining(sortedSegments, tableLevelProperties);
// filter the segments if the compaction based on days is configured.
List<LoadMetadataDetails> listOfSegmentsLoadedInSameDateInterval =
- identifySegmentsToBeMergedBasedOnLoadedDate(listOfSegmentsAfterPreserve);
+ identifySegmentsToBeMergedBasedOnLoadedDate(listOfSegmentsAfterPreserve,
+ tableLevelProperties);
List<LoadMetadataDetails> listOfSegmentsToBeMerged;
// identify the segments to merge based on the Size of the segments across partition.
if (CompactionType.MAJOR == compactionType) {
listOfSegmentsToBeMerged = identifySegmentsToBeMergedBasedOnSize(compactionSize,
- listOfSegmentsLoadedInSameDateInterval, carbonLoadModel, tablePath);
+ listOfSegmentsLoadedInSameDateInterval, carbonLoadModel, tablePath);
} else {
listOfSegmentsToBeMerged =
- identifySegmentsToBeMergedBasedOnSegCount(listOfSegmentsLoadedInSameDateInterval);
+ identifySegmentsToBeMergedBasedOnSegCount(listOfSegmentsLoadedInSameDateInterval,
+ tableLevelProperties);
}
return listOfSegmentsToBeMerged;
@@ -430,7 +432,8 @@ public final class CarbonDataMergerUtil {
public static void sortSegments(List segments) {
// sort the segment details.
Collections.sort(segments, new Comparator<LoadMetadataDetails>() {
- @Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
+ @Override
+ public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
double seg1Id = Double.parseDouble(seg1.getLoadName());
double seg2Id = Double.parseDouble(seg2.getLoadName());
return Double.compare(seg1Id, seg2Id);
@@ -443,19 +446,26 @@ public final class CarbonDataMergerUtil {
* This property is configurable.
*
* @param listOfSegmentsBelowThresholdSize
+ * @param tblProps
* @return
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnLoadedDate(
- List<LoadMetadataDetails> listOfSegmentsBelowThresholdSize) {
+ List<LoadMetadataDetails> listOfSegmentsBelowThresholdSize, Map<String, String> tblProps) {
List<LoadMetadataDetails> loadsOfSameDate =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
long numberOfDaysAllowedToMerge = 0;
try {
+ // overwrite system level option by table level option if exists
numberOfDaysAllowedToMerge = Long.parseLong(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
- CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT));
+ .getProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
+ CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT));
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS)) {
+ numberOfDaysAllowedToMerge = Long.parseLong(
+ tblProps.get(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS));
+ }
+
if (numberOfDaysAllowedToMerge < 0 || numberOfDaysAllowedToMerge > 100) {
LOGGER.error(
"The specified value for property " + CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT
@@ -675,18 +685,27 @@ public final class CarbonDataMergerUtil {
*
* @param listOfSegmentsAfterPreserve the list of segments after
* preserve and before filtering by minor compaction level
+ * @param tblProps
* @return the list of segments to be merged after filtering by minor compaction level
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSegCount(
- List<LoadMetadataDetails> listOfSegmentsAfterPreserve) {
+ List<LoadMetadataDetails> listOfSegmentsAfterPreserve, Map<String, String> tblProps) {
List<LoadMetadataDetails> mergedSegments =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<LoadMetadataDetails> unMergedSegments =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- int[] noOfSegmentLevelsCount =
- CarbonProperties.getInstance().getCompactionSegmentLevelCount();
+ new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+ int[] noOfSegmentLevelsCount = CarbonProperties.getInstance()
+ .getCompactionSegmentLevelCount();
+ // overwrite system level option by table level option if exists
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD)) {
+ noOfSegmentLevelsCount = CarbonProperties.getInstance()
+ .getIntArray(tblProps.get(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD));
+ if (0 == noOfSegmentLevelsCount.length) {
+ noOfSegmentLevelsCount = CarbonProperties.getInstance().getCompactionSegmentLevelCount();
+ }
+ }
int level1Size = 0;
int level2Size = 0;
@@ -756,14 +775,21 @@ public final class CarbonDataMergerUtil {
* checks number of loads to be preserved and returns remaining valid segments
*
* @param segments
+ * @param tblProps
* @return
*/
private static List<LoadMetadataDetails> checkPreserveSegmentsPropertyReturnRemaining(
- List<LoadMetadataDetails> segments) {
+ List<LoadMetadataDetails> segments, Map<String, String> tblProps) {
// check whether the preserving of the segments from merging is enabled or not.
- // get the number of loads to be preserved.
- int numberOfSegmentsToBePreserved =
- CarbonProperties.getInstance().getNumberOfSegmentsToBePreserved();
+ // get the number of loads to be preserved. default value is system level option
+ // overwrite system level option by table level option if exists
+ int numberOfSegmentsToBePreserved = CarbonProperties.getInstance()
+ .getNumberOfSegmentsToBePreserved();
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS)) {
+ numberOfSegmentsToBePreserved = Integer.parseInt(
+ tblProps.get(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS));
+ }
+
// get the number of valid segments and retain the latest loads from merging.
return CarbonDataMergerUtil
.getValidLoadDetailsWithRetaining(segments, numberOfSegmentsToBePreserved);
@@ -808,14 +834,23 @@ public final class CarbonDataMergerUtil {
* This will give the compaction sizes configured based on compaction type.
*
* @param compactionType
+ * @param carbonLoadModel
* @return
*/
- public static long getCompactionSize(CompactionType compactionType) {
-
+ public static long getCompactionSize(CompactionType compactionType,
+ CarbonLoadModel carbonLoadModel) {
long compactionSize = 0;
switch (compactionType) {
case MAJOR:
+ // default value is system level option
compactionSize = CarbonProperties.getInstance().getMajorCompactionSize();
+ // if table level option is identified, use it to overwrite system level option
+ Map<String, String> tblProps = carbonLoadModel.getCarbonDataLoadSchema()
+ .getCarbonTable().getTableInfo().getFactTable().getTableProperties();
+ if (tblProps.containsKey(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE)) {
+ compactionSize = Long.parseLong(
+ tblProps.get(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE));
+ }
break;
default: // this case can not come.
}
[14/50] [abbrv] carbondata git commit: [CARBONDATA-1900][Core,
processing] Modify loadmetadata to store timestamp long value(in ms)
instead of formatted date string for fields "loadStartTime" and "timestamp"
Posted by ja...@apache.org.
[CARBONDATA-1900][Core,processing] Modify loadmetadata to store timestamp long value(in ms) instead of formatted date string for fields "loadStartTime" and "timestamp"
If the table is moved to environment having different timezone or we change the system current timezone, after IUD operation some of the blocks are not treated as valid blocks.
We should stop writing the loadStartTime and timestamp in dd-MM-yyyy HH:mm:ss:SSS format. We should write the long value of the timestamp
This closes #1666
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/804ddb76
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/804ddb76
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/804ddb76
Branch: refs/heads/fgdatamap
Commit: 804ddb76fc32eee8a5ab79d8bdd4ede7c64560a1
Parents: 59de7cd
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri Dec 15 14:34:18 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue Dec 19 15:50:10 2017 +0530
----------------------------------------------------------------------
.../core/statusmanager/LoadMetadataDetails.java | 90 +++++++++++++-------
1 file changed, 60 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/804ddb76/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index bb7fc9d..85602bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -26,6 +26,29 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+/*
+ Prior to Carbon 1.3 the the loadMetaData @timestamp and @loadStartTime was stored as
+ as date string format "dd-MM-yyyy HH:mm:ss:SSS". The date string value is specific
+ to the timezone. SO the timestamp in long by the date string will not result into
+ same value if converted to date in different timezone.
+ Json Object of LoadMetaData before CarbonData 1.3
+ |-------------------------------------------------------------------------------------------|
+ | [{"timestamp":"15-12-2017 16:50:31:703","loadStatus":"Success","loadName":"0", |
+ | "partitionCount":"0","isDeleted":"FALSE","dataSize":"912","indexSize":"700", |
+ | "updateDeltaEndTimestamp":"","updateDeltaStartTimestamp":"","updateStatusFileName":"", |
+ | "loadStartTime":"15-12-2017 16:50:27:493","visibility":"true","fileFormat":"COLUMNAR_V3"}]|
+ |-------------------------------------------------------------------------------------------|
+ Fix: As the System.currentTimeMillis() returns the same value irrespective of timezone.
+ So if Carbon stores the long value for @timestamp and @loadStartTime value then the
+ value will be same irrespective of the timezone.
+ Json Object of LoadMetaData for CarbonData 1.3
+ |-------------------------------------------------------------------------------------------|
+ | [{"timestamp":"1513336827593","loadStatus":"Success","loadName":"0", |
+ | "partitionCount":"0","isDeleted":"FALSE","dataSize":"912","indexSize":"700", |
+ | "updateDeltaEndTimestamp":"","updateDeltaStartTimestamp":"","updateStatusFileName":"", |
+ | "loadStartTime":"1513336827593","visibility":"true","fileFormat":"COLUMNAR_V3"}] |
+ |-------------------------------------------------------------------------------------------|
+ */
public class LoadMetadataDetails implements Serializable {
private static final long serialVersionUID = 1106104914918491724L;
@@ -116,7 +139,7 @@ public class LoadMetadataDetails implements Serializable {
}
public void setLoadEndTime(long timestamp) {
- this.timestamp = getTimeStampConvertion(timestamp);;
+ this.timestamp = Long.toString(timestamp);
}
public SegmentStatus getSegmentStatus() {
@@ -150,7 +173,7 @@ public class LoadMetadataDetails implements Serializable {
*/
public void setModificationOrdeletionTimesStamp(long modificationOrdeletionTimesStamp) {
this.modificationOrdeletionTimesStamp =
- getTimeStampConvertion(modificationOrdeletionTimesStamp);
+ Long.toString(modificationOrdeletionTimesStamp);
}
/* (non-Javadoc)
@@ -208,28 +231,34 @@ public class LoadMetadataDetails implements Serializable {
* This method will convert a given timestamp to long value and then to string back
*
* @param factTimeStamp
- * @return
+ * @return Long TimeStamp value is milliseconds
*/
private long convertTimeStampToLong(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- Date dateToStr = null;
try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+ return Long.parseLong(factTimeStamp);
+ } catch (NumberFormatException nf) {
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
+ // it is the processing for existing table before carbon 1.3
+ Date dateToStr = null;
try {
- // if the load is in progress, factTimeStamp will be null, so use current time
- if (null == factTimeStamp) {
- return System.currentTimeMillis();
- }
dateToStr = parser.parse(factTimeStamp);
return dateToStr.getTime();
- } catch (ParseException e1) {
+ } catch (ParseException e) {
LOGGER
- .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
- return 0;
+ .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
+ parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+ try {
+ // if the load is in progress, factTimeStamp will be null, so use current time
+ if (null == factTimeStamp) {
+ return System.currentTimeMillis();
+ }
+ dateToStr = parser.parse(factTimeStamp);
+ return dateToStr.getTime();
+ } catch (ParseException e1) {
+ LOGGER.error(
+ "Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
+ return 0;
+ }
}
}
}
@@ -238,29 +267,30 @@ public class LoadMetadataDetails implements Serializable {
* returns load start time as long value
*
* @param loadStartTime
- * @return
+ * @return Long TimeStamp value is nanoseconds
*/
public Long getTimeStamp(String loadStartTime) {
- Date dateToStr = null;
try {
- dateToStr = parser.parse(loadStartTime);
- return dateToStr.getTime() * 1000;
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + loadStartTime + " to Time/Long type value" + e.getMessage());
- return null;
+ return Long.parseLong(loadStartTime) * 1000L;
+ } catch (NumberFormatException nf) {
+ // it is the processing for existing table before carbon 1.3
+ Date dateToStr = null;
+ try {
+ dateToStr = parser.parse(loadStartTime);
+ return dateToStr.getTime() * 1000;
+ } catch (ParseException e) {
+ LOGGER.error("Cannot convert" + loadStartTime +
+ " to Time/Long type value" + e.getMessage());
+ return null;
+ }
}
}
- private String getTimeStampConvertion(long time) {
- SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- return sdf.format(time);
- }
-
/**
* @param loadStartTime
*/
public void setLoadStartTime(long loadStartTime) {
- this.loadStartTime = getTimeStampConvertion(loadStartTime);
+ this.loadStartTime = Long.toString(loadStartTime);
}
/**
[28/50] [abbrv] carbondata git commit: [CARBONDATA-1899] Add
CarbonData concurrency test case
Posted by ja...@apache.org.
[CARBONDATA-1899] Add CarbonData concurrency test case
Add CarbonData concurrency test case
This closes #1670
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/71959dad
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/71959dad
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/71959dad
Branch: refs/heads/fgdatamap
Commit: 71959dad67cfc3a2542474c11f5b58d9157061ae
Parents: 47aafab
Author: xubo245 <60...@qq.com>
Authored: Fri Dec 15 22:10:00 2017 +0800
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Dec 21 12:14:40 2017 +0530
----------------------------------------------------------------------
.../carbondata/examples/ConcurrencyTest.scala | 358 +++++++++++++++++++
1 file changed, 358 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/71959dad/examples/spark2/src/main/scala/org/apache/carbondata/examples/ConcurrencyTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ConcurrencyTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ConcurrencyTest.scala
new file mode 100644
index 0000000..a0b0e1a
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ConcurrencyTest.scala
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+import java.util
+import java.util.concurrent.{Callable, Executors, Future, TimeUnit}
+
+import scala.util.Random
+
+import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+
+// scalastyle:off println
+object ConcurrencyTest {
+
+ var totalNum = 100 * 1000 * 1000
+ var ThreadNum = 16
+ var TaskNum = 100
+ var ResultIsEmpty = true
+ val cardinalityId = 10000 * 10000
+ val cardinalityCity = 6
+
+ def parquetTableName: String = "comparetest_parquet"
+
+ def orcTableName: String = "comparetest_orc"
+
+ def carbonTableName(version: String): String = s"comparetest_carbonV$version"
+
+ // Table schema:
+ // +-------------+-----------+-------------+-------------+------------+
+ // | id | string | 100,000,000 | dimension | no |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | Column name | Data type | Cardinality | Column type | Dictionary |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | city | string | 6 | dimension | yes |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | country | string | 6 | dimension | yes |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | planet | string | 100,007 | dimension | yes |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | m1 | short | NA | measure | no |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | m2 | int | NA | measure | no |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | m3 | big int | NA | measure | no |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | m4 | double | NA | measure | no |
+ // +-------------+-----------+-------------+-------------+------------+
+ // | m5 | decimal | NA | measure | no |
+ // +-------------+-----------+-------------+-------------+------------+
+
+ private def generateDataFrame(spark: SparkSession): DataFrame = {
+ val rdd = spark.sparkContext
+ .parallelize(1 to totalNum, 4)
+ .map { x =>
+ ((x % 100000000).toString, "city" + x % 6, "country" + x % 6, "planet" + x % 10007,
+ (x % 16).toShort, x / 2, (x << 1).toLong, x.toDouble / 13,
+ BigDecimal.valueOf(x.toDouble / 11))
+ }.map { x =>
+ Row(x._1, x._2, x._3, x._4, x._5, x._6, x._7, x._8, x._9)
+ }
+
+ val schema = StructType(
+ Seq(
+ StructField("id", StringType, nullable = false),
+ StructField("city", StringType, nullable = false),
+ StructField("country", StringType, nullable = false),
+ StructField("planet", StringType, nullable = false),
+ StructField("m1", ShortType, nullable = false),
+ StructField("m2", IntegerType, nullable = false),
+ StructField("m3", LongType, nullable = false),
+ StructField("m4", DoubleType, nullable = false),
+ StructField("m5", DecimalType(30, 10), nullable = false)
+ )
+ )
+
+ spark.createDataFrame(rdd, schema)
+ }
+
+ // performance test queries, they are designed to test various data access type
+ val r = new Random()
+ val tmpId = r.nextInt(cardinalityId) % totalNum
+ val tmpCity = "city" + (r.nextInt(cardinalityCity) % totalNum)
+ val queries: Array[Query] = Array(
+ Query(
+ "select * from $table" + s" where id = '$tmpId' ",
+ "filter scan",
+ "filter on high card dimension"
+ ),
+
+ Query(
+ "select id from $table" + s" where id = '$tmpId' ",
+ "filter scan",
+ "filter on high card dimension"
+ ),
+
+ Query(
+ "select * from $table" + s" where city = '$tmpCity' ",
+ "filter scan",
+ "filter on high card dimension"
+ ),
+
+ Query(
+ "select city from $table" + s" where city = '$tmpCity' ",
+ "filter scan",
+ "filter on high card dimension"
+ ),
+
+ Query(
+ "select country, sum(m1) from $table group by country",
+ "aggregate",
+ "group by on big data, on medium card column, medium result set,"
+ ),
+
+ Query(
+ "select country, sum(m1) from $table" +
+ s" where id = '$tmpId' group by country",
+ "aggregate",
+ "group by on big data, on medium card column, medium result set,"
+ ),
+
+ Query(
+ "select t1.country, sum(t1.m1) from $table t1 join $table t2"
+ + s" on t1.id = t2.id where t1.id = '$tmpId' group by t1.country",
+ "aggregate",
+ "group by on big data, on medium card column, medium result set,"
+ )
+ ,
+ Query(
+ "select t2.country, sum(t2.m1) " +
+ "from $table t1 join $table t2 join $table t3 " +
+ "join $table t4 join $table t5 join $table t6 join $table t7 " +
+ s"on t1.id=t2.id and t1.id=t3.id and t1.id=t4.id " +
+ s"and t1.id=t5.id and t1.id=t6.id and " +
+ s"t1.id=t7.id " +
+ s" where t2.id = '$tmpId' " +
+ s" group by t2.country",
+ "aggregate",
+ "group by on big data, on medium card column, medium result set,"
+ )
+ )
+
+ private def loadParquetTable(spark: SparkSession, input: DataFrame, table: String)
+ : Double = time {
+ // partitioned by last 1 digit of id column
+ val dfWithPartition = input.withColumn("partitionCol", input.col("id").%(10))
+ dfWithPartition.write
+ .partitionBy("partitionCol")
+ .mode(SaveMode.Overwrite)
+ .parquet(table)
+ spark.read.parquet(table).createOrReplaceTempView(table)
+ }
+
+ private def loadOrcTable(spark: SparkSession, input: DataFrame, table: String): Double = time {
+ // partitioned by last 1 digit of id column
+ input.write
+ .mode(SaveMode.Overwrite)
+ .orc(table)
+ spark.read.orc(table).createOrReplaceTempView(table)
+ }
+
+ private def loadCarbonTable(spark: SparkSession, input: DataFrame, tableName: String): Double = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
+ "3"
+ )
+ spark.sql(s"drop table if exists $tableName")
+ time {
+ input.write
+ .format("carbondata")
+ .option("tableName", tableName)
+ .option("tempCSV", "false")
+ .option("single_pass", "true")
+ .option("dictionary_exclude", "id") // id is high cardinality column
+ .option("table_blocksize", "32")
+ .mode(SaveMode.Overwrite)
+ .save()
+ }
+ }
+
+ // load data into parquet, carbonV2, carbonV3
+ def prepareTable(spark: SparkSession, table1: String, table2: String): Unit = {
+ val df = generateDataFrame(spark).cache
+ println(s"generating ${df.count} records, schema: ${df.schema}")
+ val table1Time = if (table1.endsWith("parquet")) {
+ loadParquetTable(spark, df, table1)
+ } else if (table1.endsWith("orc")) {
+ loadOrcTable(spark, df, table1)
+ } else {
+ sys.error("invalid table: " + table1)
+ }
+ val table2Time = loadCarbonTable(spark, df, table2)
+ println(s"load completed, time: $table1Time, $table2Time")
+ df.unpersist()
+ }
+
+ // Run all queries for the specified table
+ private def runQueries(spark: SparkSession, tableName: String): Unit = {
+ println(s"start running queries for $tableName...")
+ val start = System.currentTimeMillis()
+ println("90% time: xx.xx sec\t99% time: xx.xx sec\tlast time: xx.xx sec\t " +
+ "running query sql\taverage time: xx.xx sec\t result: show it when ResultIsEmpty is false")
+ queries.zipWithIndex.map { case (query, index) =>
+ val sqlText = query.sqlText.replace("$table", tableName)
+
+ val executorService = Executors.newFixedThreadPool(ThreadNum)
+ val tasks = new util.ArrayList[Callable[Results]]()
+
+ for (num <- 1 to TaskNum) {
+ tasks.add(new QueryTask(spark, sqlText))
+ }
+ val results = executorService.invokeAll(tasks)
+
+ val sql = s"query ${index + 1}: $sqlText "
+ printResult(results, sql)
+ executorService.shutdown()
+ executorService.awaitTermination(600, TimeUnit.SECONDS)
+
+ val taskTime = (System.currentTimeMillis() - start).toDouble / 1000
+ println("task time: " + taskTime.formatted("%.3f") + " s")
+ }
+ }
+
+ def printResult(results: util.List[Future[Results]], sql: String = "") {
+ val timeArray = new Array[Double](results.size())
+ val sqlResult = results.get(0).get().sqlResult
+ for (i <- 0 until results.size()) {
+ results.get(i).get()
+ }
+ for (i <- 0 until results.size()) {
+ timeArray(i) = results.get(i).get().time
+ }
+ val sortTimeArray = timeArray.sorted
+
+ // the time of 90 percent sql are finished
+ val time90 = ((sortTimeArray.length) * 0.9).toInt - 1
+ // the time of 99 percent sql are finished
+ val time99 = ((sortTimeArray.length) * 0.99).toInt - 1
+ print("90%:" + sortTimeArray(time90).formatted("%.3f") + " s," +
+ "\t99%:" + sortTimeArray(time99).formatted("%.3f") + " s," +
+ "\tlast:" + sortTimeArray.last.formatted("%.3f") + " s," +
+ "\t" + sql +
+ "\taverage:" + (timeArray.sum / timeArray.length).formatted("%.3f") + " s," +
+ "\t" + sqlResult.mkString(",") + "\t")
+ }
+
+ case class Results(time: Double, sqlResult: Array[Row])
+
+
+ class QueryTask(spark: SparkSession, query: String)
+ extends Callable[Results] with Serializable {
+ override def call(): Results = {
+ var result: Array[Row] = null
+ val rt = time {
+ result = spark.sql(query).head(1)
+ }
+ if (ResultIsEmpty) {
+ Results(rt, Array.empty[Row])
+ } else {
+ Results(rt, result)
+ }
+ }
+ }
+
+ // run testcases and print comparison result
+ def runTest(spark: SparkSession, table1: String, table2: String): Unit = {
+ // run queries on parquet and carbon
+ runQueries(spark, table1)
+ // do GC and sleep for some time before running next table
+ System.gc()
+ Thread.sleep(1000)
+ System.gc()
+ Thread.sleep(1000)
+ runQueries(spark, table2)
+ }
+
+ def time(code: => Unit): Double = {
+ val start = System.currentTimeMillis()
+ code
+ // return time in second
+ (System.currentTimeMillis() - start).toDouble / 1000
+ }
+
+ def initParameters(arr: Array[String]): Unit = {
+ if (arr.length > 0) {
+ totalNum = arr(0).toInt
+ }
+ if (arr.length > 1) {
+ ThreadNum = arr(1).toInt
+ }
+ if (arr.length > 2) {
+ TaskNum = arr(2).toInt
+ }
+ if (arr.length > 3) {
+ ResultIsEmpty = if (arr(3).equalsIgnoreCase("true")) {
+ true
+ } else if (arr(3).equalsIgnoreCase("false")) {
+ true
+ } else {
+ throw new Exception("error parameter, should be true or false")
+ }
+ }
+ }
+
+ def main(args: Array[String]): Unit = {
+ CarbonProperties.getInstance()
+ .addProperty("carbon.enable.vector.reader", "true")
+ .addProperty("enable.unsafe.sort", "true")
+ .addProperty("carbon.blockletgroup.size.in.mb", "32")
+ .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING, "true")
+ import org.apache.spark.sql.CarbonSession._
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/examples/spark2/target/store"
+
+ val spark = SparkSession
+ .builder()
+ .master("local[8]")
+ .enableHiveSupport()
+ .config("spark.driver.host", "127.0.0.1")
+ .getOrCreateCarbonSession(storeLocation)
+ spark.sparkContext.setLogLevel("warn")
+
+ initParameters(args)
+
+ val table1 = parquetTableName
+ val table2 = carbonTableName("3")
+ prepareTable(spark, table1, table2)
+ println("totalNum:" + totalNum + "\tThreadNum:" + ThreadNum +
+ "\tTaskNum:" + TaskNum + "\tResultIsEmpty:" + ResultIsEmpty)
+ runTest(spark, table1, table2)
+
+ CarbonUtil.deleteFoldersAndFiles(new File(table1))
+ spark.sql(s"drop table $table2")
+ spark.close()
+ }
+}
+
+// scalastyle:on println
[35/50] [abbrv] carbondata git commit: [CARBONDATA-1914][Dictionary
Cache] Cache Access Rectification.
Posted by ja...@apache.org.
[CARBONDATA-1914][Dictionary Cache] Cache Access Rectification.
This closes #1686
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f79b9ea3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f79b9ea3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f79b9ea3
Branch: refs/heads/fgdatamap
Commit: f79b9ea321ce0c9cb12214e0555f9425f5438c5a
Parents: 9659edc
Author: sounakr <so...@gmail.com>
Authored: Wed Dec 20 09:16:22 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Dec 21 23:25:55 2017 +0530
----------------------------------------------------------------------
.../core/cache/dictionary/Dictionary.java | 6 +
.../cache/dictionary/ForwardDictionary.java | 11 +
.../cache/dictionary/ReverseDictionary.java | 10 +
.../IncrementalColumnDictionaryGenerator.java | 4 +
.../testsuite/datamap/DataMapWriterSuite.scala | 132 +++----
.../spark/util/DictionaryLRUCacheTestCase.scala | 376 +++++++++++++++++++
.../loading/converter/FieldConverter.java | 5 +
.../impl/ComplexFieldConverterImpl.java | 6 +
.../impl/DictionaryFieldConverterImpl.java | 7 +
.../DirectDictionaryFieldConverterImpl.java | 6 +
.../impl/MeasureFieldConverterImpl.java | 7 +
.../impl/NonDictionaryFieldConverterImpl.java | 3 +
.../converter/impl/RowConverterImpl.java | 4 +
13 files changed, 514 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
index 7302de2..232d5f5 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
@@ -108,4 +108,10 @@ public interface Dictionary {
* This method will release the objects and set default value for primitive types
*/
void clear();
+
+ /**
+ * This method return the access count associated with the dictionary.
+ * @return
+ */
+ int getAccessCount();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
index abc95e8..bb5bb57 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
@@ -161,4 +161,15 @@ public class ForwardDictionary implements Dictionary {
columnDictionaryInfo
.getIncrementalSurrogateKeyFromDictionary(byteValuesOfFilterMembers, surrogates);
}
+
+ /**
+ * This method return the access count associated with the dictionary.
+ * @return
+ */
+ @Override public int getAccessCount() {
+ if (null != columnDictionaryInfo) {
+ return columnDictionaryInfo.getAccessCount();
+ }
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
index ff0e687..4672011 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
@@ -137,4 +137,14 @@ public class ReverseDictionary implements Dictionary {
}
}
+ /**
+ * This method return the access count associated with the dictionary.
+ * @return
+ */
+ @Override public int getAccessCount() {
+ if (null != columnReverseDictionaryInfo) {
+ return columnReverseDictionaryInfo.getAccessCount();
+ }
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
index 5eeeae4..e0feb04 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
@@ -151,6 +151,10 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege
"\n create dictionary cache: " + dictCacheTime +
"\n sort list, distinct and write: " + dictWriteTime +
"\n write sort info: " + sortIndexWriteTime);
+
+ if (isDictExists) {
+ CarbonUtil.clearDictionaryCache(dictionary);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index f73a202..04a5f9c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -68,9 +68,8 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
def buildTestData(numRows: Int): DataFrame = {
import sqlContext.implicits._
sqlContext.sparkContext.parallelize(1 to numRows)
- .map(x => ("a", "b", x))
+ .map(x => ("a" + x, "b", x))
.toDF("c1", "c2", "c3")
- .sort("c3")
}
def dropTable(): Unit = {
@@ -84,74 +83,81 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
test("test write datamap 2 pages") {
// register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- AbsoluteTableIdentifier.from(storeLocation + "/carbon1", "default", "carbon1"),
- classOf[C2DataMapFactory].getName,
- "test")
-
- val df = buildTestData(33000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon1")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
- assert(
- DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "blocklet end: 0"
- ))
- DataMapWriterSuite.callbackSeq = Seq()
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ AbsoluteTableIdentifier.from(storeLocation + "/carbon1", "default", "carbon1"),
+ classOf[C2DataMapFactory].getName,
+ "test")
+
+ val df = buildTestData(33000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon1")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
+ assert(
+ DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "blocklet end: 0"
+ ))
+ DataMapWriterSuite.callbackSeq = Seq()
}
test("test write datamap 2 blocklet") {
// register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- AbsoluteTableIdentifier.from(storeLocation + "/carbon2", "default", "carbon2"),
- classOf[C2DataMapFactory].getName,
- "test")
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ AbsoluteTableIdentifier.from(storeLocation + "/carbon2", "default", "carbon2"),
+ classOf[C2DataMapFactory].getName,
+ "test")
- CarbonProperties.getInstance()
- .addProperty("carbon.blockletgroup.size.in.mb", "1")
+ CarbonProperties.getInstance()
+ .addProperty("carbon.blockletgroup.size.in.mb", "1")
CarbonProperties.getInstance()
.addProperty("carbon.number.of.cores.while.loading",
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
-
- val df = buildTestData(300000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon2")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
- assert(
- DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "add page data: blocklet 0, page 2",
- "add page data: blocklet 0, page 3",
- "add page data: blocklet 0, page 4",
- "add page data: blocklet 0, page 5",
- "add page data: blocklet 0, page 6",
- "add page data: blocklet 0, page 7",
- "blocklet end: 0",
- "blocklet start 1",
- "add page data: blocklet 1, page 0",
- "add page data: blocklet 1, page 1",
- "blocklet end: 1"
- ))
- DataMapWriterSuite.callbackSeq = Seq()
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
+
+ val df = buildTestData(300000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon2")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .option("SORT_SCOPE","GLOBAL_SORT")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
+ assert(
+ DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "add page data: blocklet 0, page 2",
+ "add page data: blocklet 0, page 3",
+ "blocklet end: 0",
+ "blocklet start 1",
+ "add page data: blocklet 1, page 0",
+ "add page data: blocklet 1, page 1",
+ "add page data: blocklet 1, page 2",
+ "add page data: blocklet 1, page 3",
+ "blocklet end: 1",
+ "blocklet start 2",
+ "add page data: blocklet 2, page 0",
+ "add page data: blocklet 2, page 1",
+ "blocklet end: 2"
+ ))
+ DataMapWriterSuite.callbackSeq = Seq()
}
override def afterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
new file mode 100644
index 0000000..245ee7c
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.util
+
+import java.io.File
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.common.util.Spark2QueryTest
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.Spark2TestQueryExecutor
+import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
+import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+
+/**
+ * Test Case for Dictionary LRU Cache.
+ */
+class DictionaryLRUCacheTestCase extends Spark2QueryTest with BeforeAndAfterAll {
+ var spark : SparkSession = null
+ var path : String = null
+
+ def checkDictionaryAccessCount(databaseName: String, tableName: String): Unit = {
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ .lookupRelation(Option(databaseName), tableName)(Spark2TestQueryExecutor.spark)
+ .asInstanceOf[CarbonRelation].carbonTable
+ val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
+
+ val dimensions = carbonTable.getAllDimensions.asScala.toList
+ dimensions.foreach { dim =>
+ val columnIdentifier = dim.getColumnIdentifier
+ // Check the dictionary cache access.
+ val identifier: DictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(
+ absoluteTableIdentifier,
+ columnIdentifier,
+ columnIdentifier.getDataType)
+
+ val isDictExists: Boolean = CarbonUtil.isFileExistsForGivenColumn(identifier)
+ var dictionary: Dictionary = null
+ if (isDictExists) {
+ val dictCacheReverse: Cache[DictionaryColumnUniqueIdentifier, Dictionary]
+ = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY)
+ dictionary = dictCacheReverse.get(identifier)
+ assert(dictionary.getAccessCount == 1)
+ CarbonUtil.clearDictionaryCache(dictionary)
+
+ val dictCacheForward: Cache[DictionaryColumnUniqueIdentifier, Dictionary]
+ = CacheProvider.getInstance().createCache(CacheType.FORWARD_DICTIONARY)
+ dictionary = dictCacheForward.get(identifier)
+ assert(dictionary.getAccessCount == 1)
+ CarbonUtil.clearDictionaryCache(dictionary)
+ }
+ }
+ }
+
+
+ override def beforeAll {
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1")
+ .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1")
+
+ path = s"$resourcesPath/restructure/data_2000.csv"
+
+ sql("drop table if exists carbon_new1")
+ sql("drop table if exists carbon_new2")
+ sql("drop table if exists carbon_new3")
+ sql("drop table if exists carbon_new4")
+ sql("drop table if exists carbon_new5")
+ sql("drop table if exists carbon_new6")
+ sql("drop table if exists carbon_new7")
+ sql("drop table if exists carbon_new8")
+ sql("drop table if exists carbon_new9")
+ sql("drop table if exists carbon_new10")
+ }
+
+ test("test for dictionary LRU Cache for Load Single Pass") {
+
+ sql(
+ "CREATE TABLE carbon_new1 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new1 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new1 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ "CREATE TABLE carbon_new2 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new2 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new2 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ checkDictionaryAccessCount("default", "carbon_new2")
+ }
+
+ test("test for dictionary LRU Cache for Load Non Single Pass") {
+
+ sql(
+ "CREATE TABLE carbon_new3 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new3 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new3 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ "CREATE TABLE carbon_new4 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new4 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new4 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ checkDictionaryAccessCount("default", "carbon_new4")
+ }
+
+ test("test for dictionary LRU Cache for Select On Table") {
+
+ sql(
+ "CREATE TABLE carbon_new5 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new5 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new5 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql("select * from carbon_new5")
+
+ checkDictionaryAccessCount("default", "carbon_new5")
+
+
+ sql(
+ "CREATE TABLE carbon_new6 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new6 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new6 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql("select * from carbon_new6")
+
+ checkDictionaryAccessCount("default", "carbon_new6")
+ }
+
+ test("test for dictionary LRU Cache for Select With Filter On Table") {
+
+ sql(
+ "CREATE TABLE carbon_new7 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new7 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new7 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql("select * from carbon_new7 where CUST_ID > 10")
+
+ checkDictionaryAccessCount("default", "carbon_new7")
+
+
+ sql(
+ "CREATE TABLE carbon_new8 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new8 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new8 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql("select * from carbon_new8 where CUST_ID > 100")
+
+ checkDictionaryAccessCount("default", "carbon_new8")
+ }
+
+ test("test for dictionary LRU Cache for Insert Into") {
+
+ sql(
+ "CREATE TABLE carbon_new9 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME, ACTIVE_EMUI_VERSION,BIGINT_COLUMN1,Double_COLUMN1, " +
+ "Double_COLUMN2')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new9 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new9 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ sql("select * from carbon_new9 where CUST_ID > 10")
+
+ checkDictionaryAccessCount("default", "carbon_new9")
+
+
+ sql(
+ "CREATE TABLE carbon_new10 (CUST_ID INT,CUST_NAME STRING,ACTIVE_EMUI_VERSION STRING, DOB " +
+ "TIMESTAMP, DOJ TIMESTAMP, BIGINT_COLUMN1 BIGINT,BIGINT_COLUMN2 BIGINT,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 DECIMAL(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 INT) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES " +
+ "('dictionary_include'='CUST_NAME')")
+
+ sql("insert into carbon_new10 select * from carbon_new9")
+
+ checkDictionaryAccessCount("default", "carbon_new10")
+
+ sql(
+ s"LOAD DATA INPATH '$path' INTO TABLE carbon_new10 OPTIONS" +
+ "('DELIMITER'=',' , 'QUOTECHAR'='\"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE'," +
+ "'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
+ "BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2, " +
+ "INTEGER_COLUMN1')")
+
+ checkDictionaryAccessCount("default", "carbon_new10")
+
+ }
+
+
+
+ override def afterAll {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE,
+ CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT)
+ .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE,
+ CarbonCommonConstants.CARBON_MAX_LRU_CACHE_SIZE_DEFAULT)
+
+ sql("drop table if exists carbon_new1")
+ sql("drop table if exists carbon_new2")
+ sql("drop table if exists carbon_new3")
+ sql("drop table if exists carbon_new4")
+ sql("drop table if exists carbon_new5")
+ sql("drop table if exists carbon_new6")
+ sql("drop table if exists carbon_new7")
+ sql("drop table if exists carbon_new8")
+ sql("drop table if exists carbon_new9")
+ sql("drop table if exists carbon_new10")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/FieldConverter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/FieldConverter.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/FieldConverter.java
index 8a3e2eb..1ce8f9a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/FieldConverter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/FieldConverter.java
@@ -33,4 +33,9 @@ public interface FieldConverter {
* @throws CarbonDataLoadingException
*/
void convert(CarbonRow row, BadRecordLogHolder logHolder) throws CarbonDataLoadingException;
+
+ /**
+ * This method clears all the dictionary caches being acquired.
+ */
+ void clear();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/ComplexFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/ComplexFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/ComplexFieldConverterImpl.java
index 5ac832d..b26ef36 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/ComplexFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/ComplexFieldConverterImpl.java
@@ -52,6 +52,12 @@ public class ComplexFieldConverterImpl extends AbstractDictionaryFieldConverterI
}
}
+ /**
+ * Method to clear out the dictionary caches. In this instance nothing to clear.
+ */
+ @Override public void clear() {
+ }
+
@Override public void fillColumnCardinality(List<Integer> cardinality) {
genericDataType.fillCardinality(cardinality);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DictionaryFieldConverterImpl.java
index 4ac8850..1fb4086 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DictionaryFieldConverterImpl.java
@@ -123,6 +123,13 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
}
}
+ /**
+ * Method to clear out the dictionary cache.
+ */
+ @Override public void clear() {
+ CarbonUtil.clearDictionaryCache(dictionary);
+ }
+
@Override
public void fillColumnCardinality(List<Integer> cardinality) {
cardinality.add(dictionaryGenerator.size());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
index 7dcef81..b49cd90 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -89,6 +89,12 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
}
}
+ /**
+ * Method to clean the dictionary cache. In this instance nothing to clear.
+ */
+ @Override public void clear() {
+ }
+
@Override
public void fillColumnCardinality(List<Integer> cardinality) {
cardinality.add(Integer.MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
index 06f7589..2d70f03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
@@ -98,4 +98,11 @@ public class MeasureFieldConverterImpl implements FieldConverter {
}
}
+
+ /**
+ * Method to clean the dictionary cache. As in this MeasureFieldConverterImpl convert no
+ * dictionary caches are acquired so nothing to clear. s
+ */
+ @Override public void clear() {
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
index e606cdb..8f15e2e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -87,6 +87,9 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
}
}
+ @Override public void clear() {
+ }
+
private void updateWithNullValue(CarbonRow row) {
if (dataType == DataTypes.STRING) {
row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f79b9ea3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
index 959e4f5..7fc8ed3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
@@ -183,6 +183,10 @@ public class RowConverterImpl implements RowConverter {
@Override
public void finish() {
+ // Clear up dictionary cache access count.
+ for (int i = 0; i < fieldConverters.length; i ++) {
+ fieldConverters[i].clear();
+ }
// close dictionary client when finish write
if (configuration.getUseOnePass()) {
for (DictionaryClient client : dictClients) {
[05/50] [abbrv] carbondata git commit: [CARBONDATA-1895] Fix issue of
create table if not exists
Posted by ja...@apache.org.
[CARBONDATA-1895] Fix issue of create table if not exists
Fix issue of create table if not exists
This closes #1657
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9f332be2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9f332be2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9f332be2
Branch: refs/heads/fgdatamap
Commit: 9f332be245598c73e4fe81495182293dc8aa6c32
Parents: 4a7fc66
Author: chenerlu <ch...@huawei.com>
Authored: Wed Dec 13 23:36:49 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Dec 17 10:52:19 2017 +0800
----------------------------------------------------------------------
.../TestCreateTableIfNotExists.scala | 47 ++++++++
.../table/CarbonCreateTableCommand.scala | 106 +++++++++----------
2 files changed, 100 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f332be2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableIfNotExists.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableIfNotExists.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableIfNotExists.scala
new file mode 100644
index 0000000..a8fcf12
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableIfNotExists.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateTableIfNotExists extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("use default")
+ sql("drop table if exists test")
+ }
+
+ test("test create table if not exists") {
+ sql("create table test(a int, b string) stored by 'carbondata'")
+ try {
+ // table creation should be successful
+ sql("create table if not exists test(a int, b string) stored by 'carbondata'")
+ assert(true)
+ } catch {
+ case ex: Exception =>
+ assert(false)
+ }
+ }
+
+ override def afterAll {
+ sql("use default")
+ sql("drop table if exists test")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f332be2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index 78b9634..314e551 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -62,66 +62,66 @@ case class CarbonCreateTableCommand(
s"Table [$tableName] already exists under database [$dbName]")
throw new TableAlreadyExistsException(dbName, tableName)
}
- }
-
- val tablePath = tableLocation.getOrElse(
- CarbonEnv.getTablePath(Some(dbName), tableName)(sparkSession))
- tableInfo.setTablePath(tablePath)
- val tableIdentifier = AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
+ } else {
+ val tablePath = tableLocation.getOrElse(
+ CarbonEnv.getTablePath(Some(dbName), tableName)(sparkSession))
+ tableInfo.setTablePath(tablePath)
+ val tableIdentifier = AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
- // Add validation for sort scope when create table
- val sortScope = tableInfo.getFactTable.getTableProperties.asScala
- .getOrElse("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
- if (!CarbonUtil.isValidSortOption(sortScope)) {
- throw new InvalidConfigurationException(
- s"Passing invalid SORT_SCOPE '$sortScope', valid SORT_SCOPE are 'NO_SORT', 'BATCH_SORT'," +
- s" 'LOCAL_SORT' and 'GLOBAL_SORT' ")
- }
+ // Add validation for sort scope when create table
+ val sortScope = tableInfo.getFactTable.getTableProperties.asScala
+ .getOrElse("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
+ if (!CarbonUtil.isValidSortOption(sortScope)) {
+ throw new InvalidConfigurationException(
+ s"Passing invalid SORT_SCOPE '$sortScope', valid SORT_SCOPE are 'NO_SORT'," +
+ s" 'BATCH_SORT', 'LOCAL_SORT' and 'GLOBAL_SORT' ")
+ }
- if (tableInfo.getFactTable.getListOfColumns.size <= 0) {
- CarbonException.analysisException("Table should have at least one column.")
- }
+ if (tableInfo.getFactTable.getListOfColumns.size <= 0) {
+ CarbonException.analysisException("Table should have at least one column.")
+ }
- val operationContext = new OperationContext
- val createTablePreExecutionEvent: CreateTablePreExecutionEvent =
- CreateTablePreExecutionEvent(sparkSession, tableIdentifier, Some(tableInfo))
- OperationListenerBus.getInstance.fireEvent(createTablePreExecutionEvent, operationContext)
- val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val carbonSchemaString = catalog.generateTableSchemaString(tableInfo, tableIdentifier)
- if (createDSTable) {
- try {
- val tablePath = tableIdentifier.getTablePath
- val carbonRelation = CarbonSparkUtil.createCarbonRelation(tableInfo, tablePath)
- val rawSchema = CarbonSparkUtil.getRawSchema(carbonRelation)
- sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
- sparkSession.sql(
- s"""CREATE TABLE $dbName.$tableName
- |(${ rawSchema })
- |USING org.apache.spark.sql.CarbonSource
- |OPTIONS (
- | tableName "$tableName",
- | dbName "$dbName",
- | tablePath "$tablePath",
- | path "$tablePath"
- | $carbonSchemaString)
+ val operationContext = new OperationContext
+ val createTablePreExecutionEvent: CreateTablePreExecutionEvent =
+ CreateTablePreExecutionEvent(sparkSession, tableIdentifier, Some(tableInfo))
+ OperationListenerBus.getInstance.fireEvent(createTablePreExecutionEvent, operationContext)
+ val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonSchemaString = catalog.generateTableSchemaString(tableInfo, tableIdentifier)
+ if (createDSTable) {
+ try {
+ val tablePath = tableIdentifier.getTablePath
+ val carbonRelation = CarbonSparkUtil.createCarbonRelation(tableInfo, tablePath)
+ val rawSchema = CarbonSparkUtil.getRawSchema(carbonRelation)
+ sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
+ sparkSession.sql(
+ s"""CREATE TABLE $dbName.$tableName
+ |(${ rawSchema })
+ |USING org.apache.spark.sql.CarbonSource
+ |OPTIONS (
+ | tableName "$tableName",
+ | dbName "$dbName",
+ | tablePath "$tablePath",
+ | path "$tablePath"
+ | $carbonSchemaString)
""".stripMargin)
- } catch {
- case e: AnalysisException => throw e
- case e: Exception =>
- // call the drop table to delete the created table.
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .dropTable(tableIdentifier)(sparkSession)
+ } catch {
+ case e: AnalysisException => throw e
+ case e: Exception =>
+ // call the drop table to delete the created table.
+ CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .dropTable(tableIdentifier)(sparkSession)
- val msg = s"Create table'$tableName' in database '$dbName' failed."
- LOGGER.audit(msg)
- LOGGER.error(e, msg)
- CarbonException.analysisException(msg)
+ val msg = s"Create table'$tableName' in database '$dbName' failed."
+ LOGGER.audit(msg)
+ LOGGER.error(e, msg)
+ CarbonException.analysisException(msg)
+ }
}
+ val createTablePostExecutionEvent: CreateTablePostExecutionEvent =
+ CreateTablePostExecutionEvent(sparkSession, tableIdentifier)
+ OperationListenerBus.getInstance.fireEvent(createTablePostExecutionEvent, operationContext)
+ LOGGER.audit(s"Table created with Database name [$dbName] and Table name [$tableName]")
}
- val createTablePostExecutionEvent: CreateTablePostExecutionEvent =
- CreateTablePostExecutionEvent(sparkSession, tableIdentifier)
- OperationListenerBus.getInstance.fireEvent(createTablePostExecutionEvent, operationContext)
- LOGGER.audit(s"Table created with Database name [$dbName] and Table name [$tableName]")
Seq.empty
}
}
[45/50] [abbrv] carbondata git commit: [CARBONDATA-1933] Support
Spark 2.2.1 in carbon partition tables
Posted by ja...@apache.org.
[CARBONDATA-1933] Support Spark 2.2.1 in carbon partition tables
This closes #1716
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f0123e79
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f0123e79
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f0123e79
Branch: refs/heads/fgdatamap
Commit: f0123e795b3ad9f5412920fadb453db31754dcfc
Parents: 9a2159b
Author: ravipesala <ra...@gmail.com>
Authored: Fri Dec 22 14:47:53 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Dec 23 21:23:16 2017 +0800
----------------------------------------------------------------------
.../StandardPartitionTableQueryTestCase.scala | 44 +++++++-
.../carbondata/spark/util/CarbonScalaUtil.scala | 29 ++++-
.../management/CarbonLoadDataCommand.scala | 109 +++++++++++--------
.../datasources/CarbonFileFormat.scala | 7 +-
4 files changed, 136 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0123e79/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index 8670162..b3c91ae 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -16,9 +16,9 @@
*/
package org.apache.carbondata.spark.testsuite.standardpartition
-import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.execution.BatchedDataSourceScanExec
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, Row}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -63,7 +63,6 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
"select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno," +
" deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary " +
"from partitionone where empno=11 order by empno")
-
verifyPartitionInfo(frame, Seq("empno=11"))
checkAnswer(frame,
@@ -186,6 +185,43 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
}
+ test("badrecords on partition column") {
+ sql("create table badrecordsPartition(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata'")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartition options('bad_records_action'='force')")
+ sql("select count(*) from badrecordsPartition").show()
+ checkAnswer(sql("select count(*) cnt from badrecordsPartition where intfield2 is null"), Seq(Row(9)))
+ checkAnswer(sql("select count(*) cnt from badrecordsPartition where intfield2 is not null"), Seq(Row(2)))
+ }
+
+ test("badrecords fail on partition column") {
+ sql("create table badrecordsPartitionfail(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata'")
+ intercept[Exception] {
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionfail options('bad_records_action'='fail')")
+
+ }
+ }
+
+ test("badrecords ignore on partition column") {
+ sql("create table badrecordsPartitionignore(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata'")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionignore options('bad_records_action'='ignore')")
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionignore where intfield2 is null"), Seq(Row(3)))
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionignore where intfield2 is not null"), Seq(Row(2)))
+ }
+
+ test("static column partition with load command") {
+ sql(
+ """
+ | CREATE TABLE staticpartitionload (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int,projectenddate Date,doj Timestamp)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionload partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
private def verifyPartitionInfo(frame: DataFrame, partitionNames: Seq[String]) = {
val plan = frame.queryExecution.sparkPlan
@@ -209,6 +245,10 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
sql("drop table if exists partitionmany")
sql("drop table if exists partitiondate")
sql("drop table if exists partitiondateinsert")
+ sql("drop table if exists badrecordsPartition")
+ sql("drop table if exists staticpartitionload")
+ sql("drop table if exists badrecordsPartitionignore")
+ sql("drop table if exists badrecordsPartitionfail")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0123e79/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 4c405a4..9fcb98f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
+import org.apache.carbondata.processing.loading.exception.BadRecordFoundException
object CarbonScalaUtil {
def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = {
@@ -154,11 +155,24 @@ object CarbonScalaUtil {
}
}
+ /**
+ * Converts incoming value to UTF8String after converting data as per the data type.
+ * @param value Input value to convert
+ * @param dataType Datatype to convert and then convert to UTF8String
+ * @param timeStampFormat Timestamp format to convert in case of timestamp datatypes
+ * @param dateFormat DataFormat to convert incase of DateType datatype
+ * @param serializationNullFormat if this encounters in input data then data will
+ * be treated as null
+ * @param fail If it is true then any conversion error will trhow error otherwise it will be
+ * filled with ull value
+ * @return converted UTF8String
+ */
def convertToUTF8String(value: String,
dataType: DataType,
timeStampFormat: SimpleDateFormat,
dateFormat: SimpleDateFormat,
- serializationNullFormat: String): UTF8String = {
+ serializationNullFormat: String,
+ fail: Boolean): UTF8String = {
if (value == null || serializationNullFormat.equals(value)) {
return UTF8String.fromString(value)
}
@@ -171,11 +185,22 @@ object CarbonScalaUtil {
UTF8String.fromString(
DateTimeUtils.dateToString(
(dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt))
+ case ShortType => UTF8String.fromString(value.toShort.toString)
+ case IntegerType => UTF8String.fromString(value.toInt.toString)
+ case LongType => UTF8String.fromString(value.toLong.toString)
+ case DoubleType => UTF8String.fromString(value.toDouble.toString)
+ case FloatType => UTF8String.fromString(value.toFloat.toString)
+ case d: DecimalType => UTF8String
+ .fromString(new java.math.BigDecimal(value).toPlainString)
+ case BooleanType => UTF8String.fromString(value.toBoolean.toString)
case _ => UTF8String.fromString(value)
}
} catch {
case e: Exception =>
- UTF8String.fromString(value)
+ if (fail) {
+ throw e
+ }
+ UTF8String.fromString(null)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0123e79/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 5eee0bf..55c8769 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -67,7 +67,7 @@ import org.apache.carbondata.format
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
-import org.apache.carbondata.processing.loading.exception.NoRetryException
+import org.apache.carbondata.processing.loading.exception.{BadRecordFoundException, NoRetryException}
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
@@ -463,13 +463,16 @@ case class CarbonLoadDataCommand(
val logicalPlan =
sparkSession.sessionState.catalog.lookupRelation(
identifier)
- val relation = logicalPlan.collect {
- case l: LogicalRelation => l
+ val catalogTable: CatalogTable = logicalPlan.collect {
+ case l: LogicalRelation => l.catalogTable.get
case c // To make compatabile with spark 2.1 and 2.2 we need to compare classes
if (c.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
c.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
c.getClass.getName.equals(
- "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation")) => c
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation")) =>
+ CarbonReflectionUtils.getFieldOfCatalogTable(
+ "tableMeta",
+ c).asInstanceOf[CatalogTable]
}.head
// Clean up the old invalid segment data.
DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, table)
@@ -481,6 +484,11 @@ case class CarbonLoadDataCommand(
// converted to hive standard fomat to let spark understand the data to partition.
val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ val failAction =
+ carbonLoadModel.getBadRecordsAction.split(",")(1).equalsIgnoreCase(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
+ val ignoreAction =
+ carbonLoadModel.getBadRecordsAction.split(",")(1).equalsIgnoreCase("ignore")
val query: LogicalPlan = if (dataFrame.isDefined) {
var timeStampformatString = CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
val timeStampFormat = new SimpleDateFormat(timeStampformatString)
@@ -513,7 +521,7 @@ case class CarbonLoadDataCommand(
sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
// In case of update, we don't need the segmrntid column in case of partitioning
val dropAttributes = attributes.dropRight(1)
- val finalOutput = relation.output.map { attr =>
+ val finalOutput = catalogTable.schema.map { attr =>
dropAttributes.find { d =>
val index = d.name.lastIndexOf("-updatedColumn")
if (index > 0) {
@@ -548,13 +556,15 @@ case class CarbonLoadDataCommand(
StructType(carbonLoadModel.getCsvHeaderColumns.map(
StructField(_, StringType))).toAttributes
val rowDataTypes = attributes.map { attribute =>
- relation.output.find(_.name.equalsIgnoreCase(attribute.name)) match {
+ catalogTable.schema.find(_.name.equalsIgnoreCase(attribute.name)) match {
case Some(attr) => attr.dataType
case _ => StringType
}
}
val len = rowDataTypes.length
- val rdd =
+ // Fail row conversion if fail/ignore badrecord action is enabled
+ val fail = failAction || ignoreAction
+ var rdd =
new NewHadoopRDD[NullWritable, StringArrayWritable](
sparkSession.sparkContext,
classOf[CSVInputFormat],
@@ -564,55 +574,60 @@ case class CarbonLoadDataCommand(
val data = new Array[Any](len)
var i = 0
val input = value.get()
- while (i < input.length) {
- // TODO find a way to avoid double conversion of date and time.
- data(i) = CarbonScalaUtil.convertToUTF8String(
- input(i),
- rowDataTypes(i),
- timeStampFormat,
- dateFormat,
- serializationNullFormat)
- i = i + 1
+ val inputLen = Math.min(input.length, len)
+ try {
+ while (i < inputLen) {
+ // TODO find a way to avoid double conversion of date and time.
+ data(i) = CarbonScalaUtil.convertToUTF8String(
+ input(i),
+ rowDataTypes(i),
+ timeStampFormat,
+ dateFormat,
+ serializationNullFormat,
+ fail)
+ i = i + 1
+ }
+ InternalRow.fromSeq(data)
+ } catch {
+ case e: Exception =>
+ if (failAction) {
+ // It is badrecord fail case.
+ throw new BadRecordFoundException(
+ s"Data load failed due to bad record: " +
+ s"${input(i)} with datatype ${rowDataTypes(i)}")
+ } else {
+ // It is bad record ignore case
+ InternalRow.empty
+ }
}
- InternalRow.fromSeq(data)
+
}
+ // In bad record ignore case filter the empty values
+ if (ignoreAction) {
+ rdd = rdd.filter(f => f.numFields != 0)
+ }
// Only select the required columns
val output = if (partition.nonEmpty) {
- relation.output.map{ attr =>
+ catalogTable.schema.map{ attr =>
attributes.find(_.name.equalsIgnoreCase(attr.name)).get
}.filter(attr => partition.get(attr.name).isEmpty)
} else {
- relation.output.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
+ catalogTable.schema.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
}
Project(output, LogicalRDD(attributes, rdd)(sparkSession))
}
- val convertRelation = relation match {
- case l: LogicalRelation =>
- convertToLogicalRelation(
- l.catalogTable.get,
- l.output,
- l.relation.sizeInBytes,
- isOverwriteTable,
- carbonLoadModel,
- sparkSession)
- case others =>
- val catalogTable = CarbonReflectionUtils.getFieldOfCatalogTable(
- "tableMeta",
- relation).asInstanceOf[CatalogTable]
- // TODO need to find a way to avoid double lookup
- val sizeInBytes =
- CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
- catalogTable.identifier)(sparkSession).asInstanceOf[CarbonRelation].sizeInBytes
- val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
- convertToLogicalRelation(
- catalogTable,
- others.output,
- sizeInBytes,
- isOverwriteTable,
- carbonLoadModel,
- sparkSession)
- }
+ // TODO need to find a way to avoid double lookup
+ val sizeInBytes =
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
+ catalogTable.identifier)(sparkSession).asInstanceOf[CarbonRelation].sizeInBytes
+ val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
+ val convertRelation = convertToLogicalRelation(
+ catalogTable,
+ sizeInBytes,
+ isOverwriteTable,
+ carbonLoadModel,
+ sparkSession)
val convertedPlan =
CarbonReflectionUtils.getInsertIntoCommand(
convertRelation,
@@ -629,14 +644,12 @@ case class CarbonLoadDataCommand(
private def convertToLogicalRelation(
catalogTable: CatalogTable,
- output: Seq[Attribute],
sizeInBytes: Long,
overWrite: Boolean,
loadModel: CarbonLoadModel,
sparkSession: SparkSession): LogicalRelation = {
val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
- val metastoreSchema = StructType(StructType.fromAttributes(
- output).fields.map(_.copy(dataType = StringType)))
+ val metastoreSchema = StructType(catalogTable.schema.fields.map(_.copy(dataType = StringType)))
val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
val catalog = new CatalogFileIndex(
sparkSession, catalogTable, sizeInBytes)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0123e79/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index b9df7a1..4b338a4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -192,7 +192,12 @@ private class CarbonOutputWriter(path: String,
partitions.map{ p =>
val splitData = p.split("=")
if (splitData.length > 1) {
- splitData(1)
+ // NUll handling case. For null hive creates with this special name
+ if (splitData(1).equals("__HIVE_DEFAULT_PARTITION__")) {
+ null
+ } else {
+ splitData(1)
+ }
} else {
""
}
[12/50] [abbrv] carbondata git commit:
[CARBONDATA-1888][PreAggregate][Bug]Fixed compaction issue in case of
timeseries
Posted by ja...@apache.org.
[CARBONDATA-1888][PreAggregate][Bug]Fixed compaction issue in case of timeseries
Problem:Compaction is failing in case of timeseries.
Solution: This is failing because in column schema timeseries function is added as aggregate function
This closes #1648
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/54eedfe6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/54eedfe6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/54eedfe6
Branch: refs/heads/fgdatamap
Commit: 54eedfe625c59dce869148bdcb77384602733e7b
Parents: 4430178
Author: kumarvishal <ku...@gmail.com>
Authored: Tue Dec 12 23:02:20 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Dec 19 09:55:19 2017 +0530
----------------------------------------------------------------------
.../ThriftWrapperSchemaConverterImpl.java | 19 ++----
.../schema/table/column/ColumnSchema.java | 18 ++---
.../util/AbstractDataFileFooterConverter.java | 11 +--
.../timeseries/TestTimeseriesCompaction.scala | 72 ++++++++++++++++++++
.../DataRetentionConcurrencyTestCase.scala | 5 +-
.../command/carbonTableSchemaCommon.scala | 2 +-
6 files changed, 94 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 5d15bf8..2b8cfa5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -38,7 +38,6 @@ import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
-import org.apache.carbondata.core.preagg.TimeSeriesUDF;
/**
* Thrift schema to carbon schema converter and vice versa
@@ -201,10 +200,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
}
properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
}
- thriftColumnSchema.setAggregate_function(wrapperColumnSchema.getAggFunction());
- if (null != wrapperColumnSchema.getTimeSeriesFunction() && !wrapperColumnSchema
+ if (null != wrapperColumnSchema.getAggFunction() && !wrapperColumnSchema.getAggFunction()
+ .isEmpty()) {
+ thriftColumnSchema.setAggregate_function(wrapperColumnSchema.getAggFunction());
+ } else if (null != wrapperColumnSchema.getTimeSeriesFunction() && !wrapperColumnSchema
.getTimeSeriesFunction().isEmpty()) {
thriftColumnSchema.setAggregate_function(wrapperColumnSchema.getTimeSeriesFunction());
+ } else {
+ thriftColumnSchema.setAggregate_function("");
}
List<ParentColumnTableRelation> parentColumnTableRelations =
wrapperColumnSchema.getParentColumnTableRelations();
@@ -528,15 +531,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setSortColumn(true);
}
}
- if (null != externalColumnSchema.getAggregate_function().toLowerCase()) {
- if (TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION
- .contains(externalColumnSchema.getAggregate_function().toLowerCase())) {
- wrapperColumnSchema
- .setTimeSeriesFunction(externalColumnSchema.getAggregate_function().toLowerCase());
- } else {
- wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function());
- }
- }
+ wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());
List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =
externalColumnSchema.getParentColumnTableRelations();
if (null != parentColumnTableRelation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index edae4d7..edede18 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.metadata.schema.table.WritableUtil;
+import org.apache.carbondata.core.preagg.TimeSeriesUDF;
/**
* Store the information about the column meta data present the table
@@ -443,20 +444,21 @@ public class ColumnSchema implements Serializable, Writable {
return aggFunction;
}
- public void setAggFunction(String aggFunction) {
- this.aggFunction = aggFunction;
+ public void setFunction(String function) {
+ if (null == function) {
+ return;
+ }
+ if (TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION.contains(function.toLowerCase())) {
+ this.timeSeriesFunction = function;
+ } else {
+ this.aggFunction = function;
+ }
}
public String getTimeSeriesFunction() {
return timeSeriesFunction;
}
- public void setTimeSeriesFunction(String timeSeriesFunction) {
- if (null != timeSeriesFunction) {
- this.timeSeriesFunction = timeSeriesFunction;
- }
- }
-
@Override
public void write(DataOutput out) throws IOException {
out.writeShort(dataType.getId());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index f65e98d..c5f9685 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
-import org.apache.carbondata.core.preagg.TimeSeriesUDF;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockIndex;
@@ -290,15 +289,7 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setSortColumn(true);
}
}
- if (null != externalColumnSchema.getAggregate_function()) {
- if (TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION
- .contains(externalColumnSchema.getAggregate_function().toLowerCase())) {
- wrapperColumnSchema
- .setTimeSeriesFunction(externalColumnSchema.getAggregate_function().toLowerCase());
- } else {
- wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function());
- }
- }
+ wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());
List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation =
externalColumnSchema.getParentColumnTableRelations();
if (null != parentColumnTableRelation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
new file mode 100644
index 0000000..561e640
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.Matchers._
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+@Ignore
+class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
+
+ var isCompactionEnabled = false
+ override def beforeAll: Unit = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ isCompactionEnabled = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,"false").toBoolean
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ sql("drop table if exists mainTable")
+ sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
+ sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
+ }
+
+ test("test if pre-agg table is compacted with parent table minor compaction") {
+ val segmentNamesSecond = sql("show segments for table maintable_agg0_second").collect().map(_.get(0).toString)
+ segmentNamesSecond should equal (Array("3", "2", "1", "0.1", "0"))
+
+ val segmentNamesMinute = sql("show segments for table maintable_agg0_minute").collect().map(_.get(0).toString)
+ segmentNamesMinute should equal (Array("3", "2", "1", "0.1", "0"))
+
+ val segmentNamesHour = sql("show segments for table maintable_agg0_hour").collect().map(_.get(0).toString)
+ segmentNamesHour should equal (Array("3", "2", "1", "0.1", "0"))
+
+ val segmentNamesday = sql("show segments for table maintable_agg0_day").collect().map(_.get(0).toString)
+ segmentNamesday should equal (Array("3", "2", "1", "0.1", "0"))
+
+ val segmentNamesmonth = sql("show segments for table maintable_agg0_month").collect().map(_.get(0).toString)
+ segmentNamesmonth should equal (Array("3", "2", "1", "0.1", "0"))
+
+ val segmentNamesyear = sql("show segments for table maintable_agg0_year").collect().map(_.get(0).toString)
+ segmentNamesyear should equal (Array("3", "2", "1", "0.1", "0"))
+ }
+
+ override def afterAll: Unit = {
+ sql("drop table if exists mainTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, isCompactionEnabled+"")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
index 23ed377..40b3de0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
@@ -20,7 +20,8 @@ package org.apache.carbondata.spark.testsuite.dataretention
import java.util
import java.util.concurrent.{Callable, Executors}
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, Ignore}
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
@@ -51,7 +52,7 @@ class DataRetentionConcurrencyTestCase extends QueryTest with BeforeAndAfterAll
sql("drop table if exists concurrent")
}
- test("DataRetention_Concurrency_load_id") {
+ ignore("DataRetention_Concurrency_load_id") {
val tasks = new util.ArrayList[Callable[String]]()
tasks
http://git-wip-us.apache.org/repos/asf/carbondata/blob/54eedfe6/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 089b60e..ad6d876 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -429,7 +429,7 @@ class TableNewProcessor(cm: TableModel) {
columnSchema.setSortColumn(false)
if(isParentColumnRelation) {
val dataMapField = map.get.get(field).get
- columnSchema.setAggFunction(dataMapField.aggregateFunction)
+ columnSchema.setFunction(dataMapField.aggregateFunction)
val relation = dataMapField.columnTableRelation.get
val parentColumnTableRelationList = new util.ArrayList[ParentColumnTableRelation]
val relationIdentifier = new RelationIdentifier(
[07/50] [abbrv] carbondata git commit:
[CARBONDATA-1288][DictionarySecureServer] Dictionary Secure Server
Implementation. During single pass load Dictionary key creation is done
through driver and executor communication through external ports. The commun
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 58671b7..2afd040 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -32,7 +32,8 @@ import org.apache.spark.util.{CausedBy, FileUtils}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
+import org.apache.carbondata.core.dictionary.server.{DictionaryServer, NonSecureDictionaryServer}
+import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceProvider
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
@@ -47,6 +48,8 @@ import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
+import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
+import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DictionaryLoadModel}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataLoadingUtil, GlobalDictionaryUtil}
@@ -163,6 +166,18 @@ case class CarbonLoadDataCommand(
if (carbonLoadModel.isAggLoadRequest) {
carbonLoadModel.setUseOnePass(false)
}
+
+ // start dictionary server when use one pass load and dimension with DICTIONARY
+ // encoding is present.
+ val allDimensions =
+ carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getAllDimensions.asScala.toList
+ val createDictionary = allDimensions.exists {
+ carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
+ !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
+ }
+ if (!createDictionary) {
+ carbonLoadModel.setUseOnePass(false)
+ }
// Create table and metadata folders if not exist
val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
@@ -283,18 +298,25 @@ case class CarbonLoadDataCommand(
val sparkDriverHost = sparkSession.sqlContext.sparkContext.
getConf.get("spark.driver.host")
carbonLoadModel.setDictionaryServerHost(sparkDriverHost)
- // start dictionary server when use one pass load and dimension with DICTIONARY
- // encoding is present.
- val allDimensions =
- carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getAllDimensions.asScala.toList
- val createDictionary = allDimensions.exists {
- carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
- !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
- }
- val server: Option[DictionaryServer] = if (createDictionary) {
- val dictionaryServer = DictionaryServer
- .getInstance(dictionaryServerPort.toInt, carbonTable)
+
+ val carbonSecureModeDictServer = CarbonProperties.getInstance.
+ getProperty(CarbonCommonConstants.CARBON_SECURE_DICTIONARY_SERVER,
+ CarbonCommonConstants.CARBON_SECURE_DICTIONARY_SERVER_DEFAULT)
+
+ val sparkConf = sparkSession.sqlContext.sparkContext.getConf
+ // For testing.
+ // sparkConf.set("spark.authenticate", "true")
+ // sparkConf.set("spark.authenticate.secret", "secret")
+
+ val server: Option[DictionaryServer] = if (sparkConf.get("spark.authenticate", "false").
+ equalsIgnoreCase("true") && carbonSecureModeDictServer.toBoolean) {
+ val dictionaryServer = SecureDictionaryServer.getInstance(sparkConf,
+ sparkDriverHost.toString, dictionaryServerPort.toInt, carbonTable)
carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
+ carbonLoadModel.setDictionaryServerHost(dictionaryServer.getHost)
+ carbonLoadModel.setDictionaryServerSecretKey(dictionaryServer.getSecretKey)
+ carbonLoadModel.setDictionaryEncryptServerSecure(dictionaryServer.isEncryptSecureServer)
+ carbonLoadModel.setDictionaryServiceProvider(new SecureDictionaryServiceProvider())
sparkSession.sparkContext.addSparkListener(new SparkListener() {
override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
dictionaryServer.shutdown()
@@ -302,7 +324,20 @@ case class CarbonLoadDataCommand(
})
Some(dictionaryServer)
} else {
- None
+ val dictionaryServer = NonSecureDictionaryServer
+ .getInstance(dictionaryServerPort.toInt, carbonTable)
+ carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
+ carbonLoadModel.setDictionaryServerHost(dictionaryServer.getHost)
+ carbonLoadModel.setDictionaryEncryptServerSecure(false)
+ carbonLoadModel
+ .setDictionaryServiceProvider(new NonSecureDictionaryServiceProvider(dictionaryServer
+ .getPort))
+ sparkSession.sparkContext.addSparkListener(new SparkListener() {
+ override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
+ dictionaryServer.shutdown()
+ }
+ })
+ Some(dictionaryServer)
}
val loadDataFrame = if (updateModel.isDefined) {
Some(getDataFrameWithTupleID())
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 7309c91..19c8f03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.datastore.TableSpec;
+import org.apache.carbondata.core.dictionary.service.DictionaryServiceProvider;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -64,6 +65,21 @@ public class CarbonDataLoadConfiguration {
*/
private int dictionaryServerPort;
+ /**
+ * dictionary server secret key
+ */
+ private String dictionaryServerSecretKey;
+
+ /**
+ * Dictionary Service Provider.
+ */
+ private DictionaryServiceProvider dictionaryServiceProvider;
+
+ /**
+ * Secure Mode or not.
+ */
+ private Boolean dictionaryEncryptServerSecure;
+
private boolean preFetch;
private int dimensionCount;
@@ -232,6 +248,30 @@ public class CarbonDataLoadConfiguration {
this.dictionaryServerPort = dictionaryServerPort;
}
+ public String getDictionaryServerSecretKey() {
+ return dictionaryServerSecretKey;
+ }
+
+ public void setDictionaryServerSecretKey(String dictionaryServerSecretKey) {
+ this.dictionaryServerSecretKey = dictionaryServerSecretKey;
+ }
+
+ public DictionaryServiceProvider getDictionaryServiceProvider() {
+ return dictionaryServiceProvider;
+ }
+
+ public void setDictionaryServiceProvider(DictionaryServiceProvider dictionaryServiceProvider) {
+ this.dictionaryServiceProvider = dictionaryServiceProvider;
+ }
+
+ public Boolean getDictionaryEncryptServerSecure() {
+ return dictionaryEncryptServerSecure;
+ }
+
+ public void setDictionaryEncryptServerSecure(Boolean dictionaryEncryptServerSecure) {
+ this.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
+ }
+
public boolean isPreFetch() {
return preFetch;
}
@@ -310,4 +350,6 @@ public class CarbonDataLoadConfiguration {
public void setTableSpec(TableSpec tableSpec) {
this.tableSpec = tableSpec;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index ab857e4..787cb7b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -218,6 +218,9 @@ public final class DataLoadProcessBuilder {
configuration.setUseOnePass(loadModel.getUseOnePass());
configuration.setDictionaryServerHost(loadModel.getDictionaryServerHost());
configuration.setDictionaryServerPort(loadModel.getDictionaryServerPort());
+ configuration.setDictionaryServerSecretKey(loadModel.getDictionaryServerSecretKey());
+ configuration.setDictionaryEncryptServerSecure(loadModel.getDictionaryEncryptServerSecure());
+ configuration.setDictionaryServiceProvider(loadModel.getDictionaryServiceProvider());
configuration.setPreFetch(loadModel.isPreFetch());
configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
index 16c4a22..83245a3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
@@ -36,6 +36,7 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.dictionary.client.DictionaryClient;
+import org.apache.carbondata.core.dictionary.service.DictionaryOnePassService;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.processing.loading.BadRecordsLogger;
@@ -119,16 +120,22 @@ public class RowConverterImpl implements RowConverter {
"DictionaryClientPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier()
.getTableName()));
}
- Future<DictionaryClient> result = executorService.submit(new Callable<DictionaryClient>() {
- @Override
- public DictionaryClient call() throws Exception {
- Thread.currentThread().setName("Dictionary client");
- DictionaryClient dictionaryClient = new DictionaryClient();
- dictionaryClient.startClient(configuration.getDictionaryServerHost(),
- configuration.getDictionaryServerPort());
- return dictionaryClient;
- }
- });
+ DictionaryOnePassService
+ .setDictionaryServiceProvider(configuration.getDictionaryServiceProvider());
+
+ Future<DictionaryClient> result =
+ executorService.submit(new Callable<DictionaryClient>() {
+ @Override public DictionaryClient call() throws Exception {
+ Thread.currentThread().setName("Dictionary client");
+ DictionaryClient client =
+ DictionaryOnePassService.getDictionayProvider().getDictionaryClient();
+ client.startClient(configuration.getDictionaryServerSecretKey(),
+ configuration.getDictionaryServerHost(), configuration.getDictionaryServerPort(),
+ configuration.getDictionaryEncryptServerSecure());
+ return client;
+ }
+ });
+
try {
// wait for client initialization finished, or will raise null pointer exception
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 7b952e3..3031b8e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -21,10 +21,12 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
+import org.apache.carbondata.core.dictionary.service.DictionaryServiceProvider;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
+
public class CarbonLoadModel implements Serializable {
private static final long serialVersionUID = 6580168429197697465L;
@@ -147,6 +149,20 @@ public class CarbonLoadModel implements Serializable {
private int dictionaryServerPort;
/**
+ * dictionary server communication Secret Key.
+ */
+ private String dictionaryServerSecretKey;
+
+ /**
+ * dictionary service provider.
+ */
+ private DictionaryServiceProvider dictionaryServiceProvider;
+
+ /**
+ * Dictionary Secure or not.
+ */
+ private Boolean dictionaryEncryptServerSecure;
+ /**
* Pre fetch data from csv reader
*/
private boolean preFetch;
@@ -330,6 +346,15 @@ public class CarbonLoadModel implements Serializable {
this.colDictFilePath = colDictFilePath;
}
+
+ public DictionaryServiceProvider getDictionaryServiceProvider() {
+ return dictionaryServiceProvider;
+ }
+
+ public void setDictionaryServiceProvider(DictionaryServiceProvider dictionaryServiceProvider) {
+ this.dictionaryServiceProvider = dictionaryServiceProvider;
+ }
+
/**
* get copy with partition
*
@@ -366,6 +391,9 @@ public class CarbonLoadModel implements Serializable {
copy.useOnePass = useOnePass;
copy.dictionaryServerHost = dictionaryServerHost;
copy.dictionaryServerPort = dictionaryServerPort;
+ copy.dictionaryServerSecretKey = dictionaryServerSecretKey;
+ copy.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
+ copy.dictionaryServiceProvider = dictionaryServiceProvider;
copy.preFetch = preFetch;
copy.isEmptyDataBadRecord = isEmptyDataBadRecord;
copy.skipEmptyLine = skipEmptyLine;
@@ -416,6 +444,9 @@ public class CarbonLoadModel implements Serializable {
copy.useOnePass = useOnePass;
copy.dictionaryServerHost = dictionaryServerHost;
copy.dictionaryServerPort = dictionaryServerPort;
+ copy.dictionaryServerSecretKey = dictionaryServerSecretKey;
+ copy.dictionaryServiceProvider = dictionaryServiceProvider;
+ copy.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
copy.preFetch = preFetch;
copy.isEmptyDataBadRecord = isEmptyDataBadRecord;
copy.skipEmptyLine = skipEmptyLine;
@@ -468,6 +499,9 @@ public class CarbonLoadModel implements Serializable {
copyObj.useOnePass = useOnePass;
copyObj.dictionaryServerHost = dictionaryServerHost;
copyObj.dictionaryServerPort = dictionaryServerPort;
+ copyObj.dictionaryServerSecretKey = dictionaryServerSecretKey;
+ copyObj.dictionaryServiceProvider = dictionaryServiceProvider;
+ copyObj.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
copyObj.preFetch = preFetch;
copyObj.isEmptyDataBadRecord = isEmptyDataBadRecord;
copyObj.skipEmptyLine = skipEmptyLine;
@@ -722,6 +756,22 @@ public class CarbonLoadModel implements Serializable {
this.dictionaryServerPort = dictionaryServerPort;
}
+ public String getDictionaryServerSecretKey() {
+ return dictionaryServerSecretKey;
+ }
+
+ public void setDictionaryServerSecretKey(String dictionaryServerSecretKey) {
+ this.dictionaryServerSecretKey = dictionaryServerSecretKey;
+ }
+
+ public Boolean getDictionaryEncryptServerSecure() {
+ return dictionaryEncryptServerSecure;
+ }
+
+ public void setDictionaryEncryptServerSecure(Boolean dictionaryEncryptServerSecure) {
+ this.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
+ }
+
public String getDictionaryServerHost() {
return dictionaryServerHost;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 3df78b3..02a076e 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -26,13 +26,16 @@ import org.apache.spark.sql.execution.streaming.{CarbonAppendableStreamSink, Sin
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
+import org.apache.carbondata.core.dictionary.server.{DictionaryServer, NonSecureDictionaryServer}
+import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceProvider
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.hadoop.streaming.CarbonStreamOutputFormat
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
+import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
+import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.util.DataLoadingUtil
import org.apache.carbondata.streaming.segment.StreamSegment
@@ -61,10 +64,9 @@ object StreamSinkFactory {
val server = startDictionaryServer(
sparkSession,
carbonTable,
- carbonLoadModel.getDictionaryServerPort)
+ carbonLoadModel)
if (server.isDefined) {
carbonLoadModel.setUseOnePass(true)
- carbonLoadModel.setDictionaryServerPort(server.get.getPort)
} else {
carbonLoadModel.setUseOnePass(false)
}
@@ -118,7 +120,7 @@ object StreamSinkFactory {
def startDictionaryServer(
sparkSession: SparkSession,
carbonTable: CarbonTable,
- port: Int): Option[DictionaryServer] = {
+ carbonLoadModel: CarbonLoadModel): Option[DictionaryServer] = {
// start dictionary server when use one pass load and dimension with DICTIONARY
// encoding is present.
val allDimensions = carbonTable.getAllDimensions.asScala.toList
@@ -126,14 +128,46 @@ object StreamSinkFactory {
carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
!carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
}
+ val carbonSecureModeDictServer = CarbonProperties.getInstance.
+ getProperty(CarbonCommonConstants.CARBON_SECURE_DICTIONARY_SERVER,
+ CarbonCommonConstants.CARBON_SECURE_DICTIONARY_SERVER_DEFAULT)
+
+ val sparkConf = sparkSession.sqlContext.sparkContext.getConf
+ val sparkDriverHost = sparkSession.sqlContext.sparkContext.
+ getConf.get("spark.driver.host")
+
val server: Option[DictionaryServer] = if (createDictionary) {
- val dictionaryServer = DictionaryServer.getInstance(port, carbonTable)
- sparkSession.sparkContext.addSparkListener(new SparkListener() {
- override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
- dictionaryServer.shutdown()
- }
- })
- Some(dictionaryServer)
+ if (sparkConf.get("spark.authenticate", "false").equalsIgnoreCase("true") &&
+ carbonSecureModeDictServer.toBoolean) {
+ val dictionaryServer = SecureDictionaryServer.getInstance(sparkConf,
+ sparkDriverHost.toString, carbonLoadModel.getDictionaryServerPort, carbonTable)
+ carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
+ carbonLoadModel.setDictionaryServerHost(dictionaryServer.getHost)
+ carbonLoadModel.setDictionaryServerSecretKey(dictionaryServer.getSecretKey)
+ carbonLoadModel.setDictionaryEncryptServerSecure(dictionaryServer.isEncryptSecureServer)
+ carbonLoadModel.setDictionaryServiceProvider(new SecureDictionaryServiceProvider())
+ sparkSession.sparkContext.addSparkListener(new SparkListener() {
+ override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
+ dictionaryServer.shutdown()
+ }
+ })
+ Some(dictionaryServer)
+ } else {
+ val dictionaryServer = NonSecureDictionaryServer
+ .getInstance(carbonLoadModel.getDictionaryServerPort, carbonTable)
+ carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
+ carbonLoadModel.setDictionaryServerHost(dictionaryServer.getHost)
+ carbonLoadModel.setDictionaryEncryptServerSecure(false)
+ carbonLoadModel
+ .setDictionaryServiceProvider(new NonSecureDictionaryServiceProvider(dictionaryServer
+ .getPort))
+ sparkSession.sparkContext.addSparkListener(new SparkListener() {
+ override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
+ dictionaryServer.shutdown()
+ }
+ })
+ Some(dictionaryServer)
+ }
} else {
None
}
[46/50] [abbrv] carbondata git commit: [CARBONDATA-1777] Added check
to refresh table if catalog relation is present in plan
Posted by ja...@apache.org.
[CARBONDATA-1777] Added check to refresh table if catalog relation is present in plan
Analysis: In spark 2.2 while doing lookup relation there was no case to handle CatalogRelation due to which the tables were not getting refreshed in different sessions.
Solution: Add case for CatalogRelation so that the table being referred is refreshed.
This closes #1692
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7997f226
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7997f226
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7997f226
Branch: refs/heads/fgdatamap
Commit: 7997f226bab2bf78d50c31402df2c9cb62101b5f
Parents: f0123e7
Author: kunal642 <ku...@gmail.com>
Authored: Wed Dec 20 14:05:06 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Dec 23 21:29:10 2017 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/CarbonEnv.scala | 2 +-
.../datamap/CarbonDropDataMapCommand.scala | 2 +-
.../CarbonAlterTableDropPartitionCommand.scala | 3 +-
.../CarbonAlterTableSplitPartitionCommand.scala | 2 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 15 ++++-----
.../spark/sql/hive/CarbonHiveMetaStore.scala | 7 +++--
.../apache/spark/sql/hive/CarbonMetaStore.scala | 2 +-
.../org/apache/spark/util/CleanFiles.scala | 3 +-
.../org/apache/spark/util/Compaction.scala | 3 +-
.../apache/spark/util/DeleteSegmentByDate.scala | 7 +++--
.../apache/spark/util/DeleteSegmentById.scala | 4 +--
.../org/apache/spark/util/TableLoader.scala | 3 +-
.../src/main/spark2.1/CarbonSessionState.scala | 2 +-
.../src/main/spark2.2/CarbonSessionState.scala | 33 ++++++++++----------
14 files changed, 48 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 6317177..d9c50d0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -133,7 +133,7 @@ object CarbonEnv {
val databaseName = getDatabaseName(databaseNameOp)(sparkSession)
val catalog = getInstance(sparkSession).carbonMetastore
// refresh cache
- catalog.checkSchemasModifiedTimeAndReloadTables()
+ catalog.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, databaseNameOp))
// try to get it from catch, otherwise lookup in catalog
catalog.getTableFromMetadataCache(databaseName, tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 9a71523..7f68b05 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -62,7 +62,7 @@ case class CarbonDropDataMapCommand(
val tablePath = CarbonEnv.getTablePath(databaseNameOp, tableName)(sparkSession)
val tableIdentifier =
AbsoluteTableIdentifier.from(tablePath, dbName.toLowerCase, tableName.toLowerCase)
- catalog.checkSchemasModifiedTimeAndReloadTables()
+ catalog.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
try {
locksToBeAcquired foreach {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index fb515fa..d85e064 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.{Executors, ExecutorService, Future}
import scala.collection.JavaConverters._
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.util.AlterTableUtil
@@ -60,7 +61,7 @@ case class CarbonAlterTableDropPartitionCommand(
val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation]
val tablePath = relation.carbonTable.getTablePath
- carbonMetaStore.checkSchemasModifiedTimeAndReloadTables()
+ carbonMetaStore.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index 1a535fd..65a0af3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -67,7 +67,7 @@ case class CarbonAlterTableSplitPartitionCommand(
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
}
- carbonMetaStore.checkSchemasModifiedTimeAndReloadTables()
+ carbonMetaStore.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
if (null == CarbonMetadata.getInstance.getCarbonTable(dbName, tableName)) {
LOGGER.error(s"Alter table failed. table not found: $dbName.$tableName")
sys.error(s"Alter table failed. table not found: $dbName.$tableName")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index ba222e2..719f571 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -440,7 +440,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
if (FileFactory.isFileExist(metadataFilePath, fileType)) {
// while drop we should refresh the schema modified time so that if any thing has changed
// in the other beeline need to update.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
removeTableFromMetadata(dbName, tableName)
updateSchemasUpdatedTime(touchSchemaFileSystemTime())
@@ -504,19 +504,20 @@ class CarbonFileMetastore extends CarbonMetaStore {
.getLastModifiedTime
}
- def checkSchemasModifiedTimeAndReloadTables() {
+ def checkSchemasModifiedTimeAndReloadTable(tableIdentifier: TableIdentifier): Boolean = {
val (timestampFile, timestampFileType) = getTimestampFileAndType()
+ var isRefreshed = false
if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
if (!(FileFactory.getCarbonFile(timestampFile, timestampFileType).
getLastModifiedTime ==
tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) {
- refreshCache()
+ metadata.carbonTables = metadata.carbonTables.filterNot(
+ table => table.getTableName.equalsIgnoreCase(tableIdentifier.table) &&
+ table.getDatabaseName.equalsIgnoreCase(tableIdentifier.database.getOrElse("default")))
+ isRefreshed = true
}
}
- }
-
- private def refreshCache() {
- metadata.carbonTables.clear()
+ isRefreshed
}
override def isReadFromHiveMetaStore: Boolean = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index 066cb1c..d5ac5ae 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -75,7 +75,7 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
// clear driver B-tree and dictionary cache
ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
}
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
removeTableFromMetadata(dbName, tableName)
CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
// discard cached table info in cachedDataSourceTables
@@ -84,8 +84,9 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
DataMapStoreManager.getInstance().clearDataMaps(absoluteTableIdentifier)
}
- override def checkSchemasModifiedTimeAndReloadTables() {
- // do nothing now
+ override def checkSchemasModifiedTimeAndReloadTable(tableIdentifier: TableIdentifier): Boolean = {
+ // do nothing
+ false
}
override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index cc0e6ab..7dc3d3f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -136,7 +136,7 @@ trait CarbonMetaStore {
def updateAndTouchSchemasUpdatedTime()
- def checkSchemasModifiedTimeAndReloadTables()
+ def checkSchemasModifiedTimeAndReloadTable(tableIdentifier: TableIdentifier): Boolean
def isReadFromHiveMetaStore: Boolean
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index c05c0f1..eba7dcd 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -18,6 +18,7 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.carbondata.api.CarbonStore
@@ -58,7 +59,7 @@ object CleanFiles {
}
val spark = TableAPIUtil.spark(storePath, s"CleanFiles: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
cleanFiles(spark, dbName, tableName, storePath, forceTableClean)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala b/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
index 200a926..d4ec81e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
@@ -17,6 +17,7 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.AlterTableModel
import org.apache.spark.sql.execution.command.management.CarbonAlterTableCompactionCommand
import org.apache.spark.sql.util.CarbonException
@@ -58,7 +59,7 @@ object Compaction {
val compactionType = TableAPIUtil.escape(args(2))
val spark = TableAPIUtil.spark(storePath, s"Compaction: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
compaction(spark, dbName, tableName, compactionType)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
index d682b21..fcc9f2f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
@@ -14,9 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
- package org.apache.spark.util
+package org.apache.spark.util
- import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.carbondata.api.CarbonStore
@@ -45,7 +46,7 @@ object DeleteSegmentByDate {
val dateValue = TableAPIUtil.escape(args(2))
val spark = TableAPIUtil.spark(storePath, s"DeleteSegmentByDate: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
deleteSegmentByDate(spark, dbName, tableName, dateValue)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
index 5b58c8d..13ef933 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
@@ -17,7 +17,7 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
-import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.carbondata.api.CarbonStore
@@ -51,7 +51,7 @@ object DeleteSegmentById {
val segmentIds = extractSegmentIds(TableAPIUtil.escape(args(2)))
val spark = TableAPIUtil.spark(storePath, s"DeleteSegmentById: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
deleteSegmentById(spark, dbName, tableName, segmentIds)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
index d439cb1..80f781e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
@@ -24,6 +24,7 @@ import scala.collection.{immutable, mutable}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -81,7 +82,7 @@ object TableLoader {
val spark = TableAPIUtil.spark(storePath, s"TableLoader: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
loadTable(spark, Option(dbName), tableName, inputPaths, map)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
index b55a6aa..a6f28c9 100644
--- a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
@@ -108,7 +108,7 @@ class CarbonSessionCatalog(
var isRefreshed = false
val storePath = CarbonProperties.getStorePath
carbonEnv.carbonMetastore.
- checkSchemasModifiedTimeAndReloadTables()
+ checkSchemasModifiedTimeAndReloadTable(identifier)
val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7997f226/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
index c951e5e..a722cbf 100644
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
@@ -41,6 +41,7 @@ import org.apache.spark.sql.internal.{SQLConf, SessionState}
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
import org.apache.spark.sql.types.DecimalType
+import org.apache.spark.util.CarbonReflectionUtils
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
@@ -90,23 +91,15 @@ class CarbonSessionCatalog(
}
- private def refreshRelationFromCache(identifier: TableIdentifier,
- carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
+ private def refreshRelationFromCache(identifier: TableIdentifier): Boolean = {
var isRefreshed = false
- val storePath = CarbonProperties.getStorePath
- carbonEnv.carbonMetastore.checkSchemasModifiedTimeAndReloadTables()
-
- val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
- carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
- carbonDatasourceHadoopRelation.carbonTable.getTableName)
- if (table.isEmpty || (table.isDefined && table.get.getTableLastUpdatedTime !=
- carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
+ if (carbonEnv.carbonMetastore.checkSchemasModifiedTimeAndReloadTable(identifier)) {
refreshTable(identifier)
DataMapStoreManager.getInstance().
- clearDataMaps(AbsoluteTableIdentifier.from(storePath,
+ clearDataMaps(AbsoluteTableIdentifier.from(CarbonProperties.getStorePath,
identifier.database.getOrElse("default"), identifier.table))
- isRefreshed = true
logInfo(s"Schema changes have been detected for table: $identifier")
+ isRefreshed = true
}
isRefreshed
}
@@ -117,10 +110,18 @@ class CarbonSessionCatalog(
var toRefreshRelation = false
rtnRelation match {
case SubqueryAlias(_,
- LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _)) =>
- toRefreshRelation = refreshRelationFromCache(name, carbonDatasourceHadoopRelation)
- case LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = refreshRelationFromCache(name, carbonDatasourceHadoopRelation)
+ LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
+ toRefreshRelation = refreshRelationFromCache(name)
+ case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
+ toRefreshRelation = refreshRelationFromCache(name)
+ case SubqueryAlias(_, relation) if
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
+ relation.getClass.getName.equals(
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
+ val catalogTable = CarbonReflectionUtils.getFieldOfCatalogTable("tableMeta",
+ relation).asInstanceOf[CatalogTable]
+ toRefreshRelation = refreshRelationFromCache(catalogTable.identifier)
case _ =>
}
[44/50] [abbrv] carbondata git commit: [CARBONDATA-1674] Describe
formatted shows partition type
Posted by ja...@apache.org.
[CARBONDATA-1674] Describe formatted shows partition type
On desc formatted command list partition should show
This closes #1689
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9a2159ba
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9a2159ba
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9a2159ba
Branch: refs/heads/fgdatamap
Commit: 9a2159ba60b8abf3354fa8e3aa4fb7c5550e1afd
Parents: 1381c9f
Author: Jatin <ja...@knoldus.in>
Authored: Fri Dec 15 10:02:12 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Dec 23 21:16:18 2017 +0800
----------------------------------------------------------------------
.../spark/testsuite/partition/TestShowPartitions.scala | 12 ++++++++++--
.../command/table/CarbonDescribeFormattedCommand.scala | 3 +++
2 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a2159ba/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
index 57739d6..af3f40c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala
@@ -19,10 +19,10 @@ package org.apache.carbondata.spark.testsuite.partition
import java.sql.Timestamp
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, DataFrame, Row}
import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
-import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
@@ -150,6 +150,14 @@ class TestShowPartition extends QueryTest with BeforeAndAfterAll {
}
+ test("show partition table: desc formatted should show partition type"){
+ //check for partition type exist in desc formatted
+ val result:DataFrame = sql("describe formatted hashTable")
+ checkExistence(result,true,"Partition Type")
+ val row: Array[Row] = result.collect().filter{row: Row => row.getString(0).contains("Partition Type")}
+ assert(row(0).getString(1).contains("HASH"))
+ }
+
test("show partition table: range partition") {
// EqualTo
checkAnswer(sql("show partitions rangeTable"), Seq(Row("0, doj = DEFAULT"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a2159ba/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index ccdbd6e..5d19e0a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -154,6 +154,9 @@ private[sql] case class CarbonDescribeFormattedCommand(
results ++=
Seq(("Partition Columns", carbonTable.getPartitionInfo(carbonTable.getTableName)
.getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
+ results ++=
+ Seq(("Partition Type", carbonTable.getPartitionInfo(carbonTable.getTableName)
+ .getPartitionType.toString, ""))
}
results.map {
case (name, dataType, null) =>
[24/50] [abbrv] carbondata git commit:
[CARBONDATA-1859][CARBONDATA-1861][PARTITION] Support show and drop
partitions
Posted by ja...@apache.org.
[CARBONDATA-1859][CARBONDATA-1861][PARTITION] Support show and drop partitions
This closes #1674
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3ff55a2e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3ff55a2e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3ff55a2e
Branch: refs/heads/fgdatamap
Commit: 3ff55a2eef1e2bc0e61ee909b62d485e5240243a
Parents: 64b2651
Author: ravipesala <ra...@gmail.com>
Authored: Sat Dec 16 22:38:00 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Dec 20 23:46:05 2017 +0530
----------------------------------------------------------------------
.../indexstore/BlockletDataMapIndexStore.java | 15 +-
.../blockletindex/BlockletDataMap.java | 26 ++-
.../blockletindex/BlockletDataMapModel.java | 13 +-
.../core/metadata/PartitionMapFileStore.java | 103 ++++++++++-
.../hadoop/api/CarbonOutputCommitter.java | 11 +-
.../hadoop/api/CarbonTableInputFormat.java | 7 +-
.../hadoop/ft/CarbonOutputMapperTest.java | 5 +-
.../StandardPartitionTableDropTestCase.scala | 172 +++++++++++++++++++
.../spark/rdd/CarbonDropPartitionRDD.scala | 140 +++++++++++++++
.../org/apache/spark/sql/CarbonCountStar.scala | 14 +-
.../command/mutation/DeleteExecution.scala | 12 +-
.../CarbonStandardAlterTableDropPartition.scala | 142 +++++++++++++++
.../table/CarbonCreateTableCommand.scala | 10 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../sql/execution/strategy/DDLStrategy.scala | 25 ++-
.../iterator/CarbonOutputIteratorWrapper.java | 4 +-
16 files changed, 660 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 59af50b..f3e55e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -72,9 +72,9 @@ public class BlockletDataMapIndexStore
BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
if (dataMap == null) {
try {
- String segmentPath = CarbonTablePath
- .getSegmentPath(identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId());
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getAbsoluteTableIdentifier().getTablePath(),
+ identifier.getSegmentId());
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
indexFileStore.readAllIIndexOfSegment(segmentPath);
PartitionMapFileStore partitionFileStore = new PartitionMapFileStore();
@@ -111,9 +111,9 @@ public class BlockletDataMapIndexStore
segmentIndexFileStoreMap.get(identifier.getSegmentId());
PartitionMapFileStore partitionFileStore =
partitionFileStoreMap.get(identifier.getSegmentId());
- String segmentPath = CarbonTablePath
- .getSegmentPath(identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId());
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getAbsoluteTableIdentifier().getTablePath(),
+ identifier.getSegmentId());
if (indexFileStore == null) {
indexFileStore = new SegmentIndexFileStore();
indexFileStore.readAllIIndexOfSegment(segmentPath);
@@ -185,7 +185,8 @@ public class BlockletDataMapIndexStore
dataMap = new BlockletDataMap();
dataMap.init(new BlockletDataMapModel(identifier.getFilePath(),
indexFileStore.getFileData(identifier.getCarbonIndexFileName()),
- partitionFileStore.getPartitions(identifier.getCarbonIndexFileName())));
+ partitionFileStore.getPartitions(identifier.getCarbonIndexFileName()),
+ partitionFileStore.isPartionedSegment()));
lruCache.put(identifier.getUniqueTableSegmentIdentifier(), dataMap,
dataMap.getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 70bae32..f1188cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -105,6 +105,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
private int[] columnCardinality;
+ private boolean isPartitionedSegment;
+
@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
long startTime = System.currentTimeMillis();
@@ -113,6 +115,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
List<DataFileFooter> indexInfo = fileFooterConverter
.getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
+ isPartitionedSegment = blockletDataMapInfo.isPartitionedSegment();
DataMapRowImpl summaryRow = null;
for (DataFileFooter fileFooter : indexInfo) {
List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
@@ -394,6 +397,14 @@ public class BlockletDataMap implements DataMap, Cacheable {
new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
}
+ /**
+ * Creates the schema to store summary information or the information which can be stored only
+ * once per datamap. It stores datamap level max/min of each column and partition information of
+ * datamap
+ * @param segmentProperties
+ * @param partitions
+ * @throws MemoryException
+ */
private void createSummarySchema(SegmentProperties segmentProperties, List<String> partitions)
throws MemoryException {
List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>(2);
@@ -518,8 +529,15 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
@Override public List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions) {
+ // First get the partitions which are stored inside datamap.
List<String> storedPartitions = getPartitions();
- if (storedPartitions != null && storedPartitions.size() > 0 && filterExp != null) {
+ // if it has partitioned datamap but there is no partitioned information stored, it means
+ // partitions are dropped so return empty list.
+ if (isPartitionedSegment && (storedPartitions == null || storedPartitions.size() == 0)) {
+ return new ArrayList<>();
+ }
+ if (storedPartitions != null && storedPartitions.size() > 0) {
+ // Check the exact match of partition information inside the stored partitions.
boolean found = false;
if (partitions != null && partitions.size() > 0) {
found = partitions.containsAll(storedPartitions);
@@ -528,6 +546,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
return new ArrayList<>();
}
}
+ // Prune with filters if the partitions are existed in this datamap
return prune(filterExp);
}
@@ -705,16 +724,17 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
private List<String> getPartitions() {
- List<String> partitions = new ArrayList<>();
DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
if (unsafeRow.getColumnCount() > 2) {
+ List<String> partitions = new ArrayList<>();
DataMapRow row = unsafeRow.getRow(2);
for (int i = 0; i < row.getColumnCount(); i++) {
partitions.add(
new String(row.getByteArray(i), CarbonCommonConstants.DEFAULT_CHARSET_CLASS));
}
+ return partitions;
}
- return partitions;
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index c98ef33..704e0f7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -20,16 +20,23 @@ import java.util.List;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
+/**
+ * It is the model object to keep the information to build or initialize BlockletDataMap.
+ */
public class BlockletDataMapModel extends DataMapModel {
private byte[] fileData;
private List<String> partitions;
- public BlockletDataMapModel(String filePath, byte[] fileData, List<String> partitions) {
+ private boolean partitionedSegment;
+
+ public BlockletDataMapModel(String filePath, byte[] fileData, List<String> partitions,
+ boolean partitionedSegment) {
super(filePath);
this.fileData = fileData;
this.partitions = partitions;
+ this.partitionedSegment = partitionedSegment;
}
public byte[] getFileData() {
@@ -39,4 +46,8 @@ public class BlockletDataMapModel extends DataMapModel {
public List<String> getPartitions() {
return partitions;
}
+
+ public boolean isPartitionedSegment() {
+ return partitionedSegment;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 853c729..8578cfe 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -25,6 +25,8 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -47,6 +49,8 @@ import com.google.gson.Gson;
public class PartitionMapFileStore {
private Map<String, List<String>> partitionMap = new HashMap<>();
+
+ private boolean partionedSegment = false;
/**
* Write partitionmapp file to the segment folder with indexfilename and corresponding partitions.
*
@@ -105,9 +109,9 @@ public class PartitionMapFileStore {
* @param segmentPath
* @throws IOException
*/
- public void mergePartitionMapFiles(String segmentPath) throws IOException {
+ public void mergePartitionMapFiles(String segmentPath, String mergeFileName) throws IOException {
CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
- if (partitionFiles != null && partitionFiles.length > 1) {
+ if (partitionFiles != null && partitionFiles.length > 0) {
PartitionMapper partitionMapper = null;
for (CarbonFile file : partitionFiles) {
PartitionMapper localMapper = readPartitionMap(file.getAbsolutePath());
@@ -119,10 +123,12 @@ public class PartitionMapFileStore {
}
}
if (partitionMapper != null) {
- String path = segmentPath + "/" + "mergedpartitions" + CarbonTablePath.PARTITION_MAP_EXT;
+ String path = segmentPath + "/" + mergeFileName + CarbonTablePath.PARTITION_MAP_EXT;
writePartitionFile(partitionMapper, path);
for (CarbonFile file : partitionFiles) {
- FileFactory.deleteAllCarbonFilesOfDir(file);
+ if (!FileFactory.deleteAllCarbonFilesOfDir(file)) {
+ throw new IOException("Old partition map files cannot be deleted");
+ }
}
}
}
@@ -146,7 +152,7 @@ public class PartitionMapFileStore {
* @param partitionMapPath
* @return
*/
- public PartitionMapper readPartitionMap(String partitionMapPath) {
+ private PartitionMapper readPartitionMap(String partitionMapPath) {
Gson gsonObjectToRead = new Gson();
DataInputStream dataInputStream = null;
BufferedReader buffReader = null;
@@ -176,9 +182,92 @@ public class PartitionMapFileStore {
public void readAllPartitionsOfSegment(String segmentPath) {
CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
if (partitionFiles != null && partitionFiles.length > 0) {
+ partionedSegment = true;
+ int i = 0;
+ // Get the latest partition map file based on the timestamp of that file.
+ long [] partitionTimestamps = new long[partitionFiles.length];
for (CarbonFile file : partitionFiles) {
- PartitionMapper partitionMapper = readPartitionMap(file.getAbsolutePath());
- partitionMap.putAll(partitionMapper.getPartitionMap());
+ partitionTimestamps[i++] =
+ Long.parseLong(file.getName().substring(
+ 0, file.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
+ }
+ Arrays.sort(partitionTimestamps);
+ PartitionMapper partitionMapper = readPartitionMap(
+ segmentPath + "/" + partitionTimestamps[partitionTimestamps.length - 1]
+ + CarbonTablePath.PARTITION_MAP_EXT);
+ partitionMap.putAll(partitionMapper.getPartitionMap());
+ }
+ }
+
+ public boolean isPartionedSegment() {
+ return partionedSegment;
+ }
+
+ /**
+ * Drops the partitions from the partition mapper file of the segment and writes to a new file.
+ * @param segmentPath
+ * @param partitionsToDrop
+ * @param uniqueId
+ * @throws IOException
+ */
+ public void dropPartitions(String segmentPath, List<String> partitionsToDrop, String uniqueId)
+ throws IOException {
+ readAllPartitionsOfSegment(segmentPath);
+ List<String> indexesToDrop = new ArrayList<>();
+ for (Map.Entry<String, List<String>> entry: partitionMap.entrySet()) {
+ for (String partition: partitionsToDrop) {
+ if (entry.getValue().contains(partition)) {
+ indexesToDrop.add(entry.getKey());
+ }
+ }
+ }
+ if (indexesToDrop.size() > 0) {
+ // Remove the indexes from partition map
+ for (String indexToDrop : indexesToDrop) {
+ partitionMap.remove(indexToDrop);
+ }
+ PartitionMapper mapper = new PartitionMapper();
+ mapper.setPartitionMap(partitionMap);
+ String path = segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT;
+ writePartitionFile(mapper, path);
+ }
+ }
+
+ /**
+ * It deletes the old partition mapper files in case of success. And in case of failure it removes
+ * the old new file.
+ * @param segmentPath
+ * @param uniqueId
+ * @param success
+ */
+ public void commitPartitions(String segmentPath, final String uniqueId, boolean success) {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ // write partition info to new file.
+ if (carbonFile.exists()) {
+ CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
+ }
+ });
+ CarbonFile latestFile = null;
+ for (CarbonFile mapFile: carbonFiles) {
+ if (mapFile.getName().startsWith(uniqueId)) {
+ latestFile = mapFile;
+ }
+ }
+ if (latestFile != null) {
+ for (CarbonFile mapFile : carbonFiles) {
+ if (latestFile != mapFile) {
+ // Remove old files in case of success scenario
+ if (success) {
+ mapFile.delete();
+ }
+ }
+ }
+ }
+ // If it is failure scenario then remove the new file.
+ if (!success && latestFile != null) {
+ latestFile.delete();
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index ec42adf..08fd1ac 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.metadata.PartitionMapFileStore;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;
@@ -75,16 +76,18 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
super.commitJob(context);
boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ String segmentPath =
+ CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
+ // Merge all partition files into a single file.
+ new PartitionMapFileStore().mergePartitionMapFiles(segmentPath,
+ loadModel.getFactTimeStamp() + "");
LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
CarbonLoaderUtil.populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS,
loadModel.getFactTimeStamp(), true);
CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(),
loadModel.getCarbonDataLoadSchema().getCarbonTable());
CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
- String segmentPath =
- CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
- // Merge all partition files into a single file.
- new PartitionMapFileStore().mergePartitionMapFiles(segmentPath);
+ new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 7db9c0a..6a2349a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
@@ -918,8 +917,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @throws IOException
* @throws KeyGenException
*/
- public BlockMappingVO getBlockRowCount(Job job, AbsoluteTableIdentifier identifier)
- throws IOException, KeyGenException {
+ public BlockMappingVO getBlockRowCount(Job job, AbsoluteTableIdentifier identifier,
+ List<String> partitions) throws IOException {
TableDataMap blockletMap = DataMapStoreManager.getInstance()
.getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
@@ -931,7 +930,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// TODO: currently only batch segment is supported, add support for streaming table
List<String> filteredSegment = getFilteredSegment(job, allSegments.getValidSegments());
- List<ExtendedBlocklet> blocklets = blockletMap.prune(filteredSegment, null, null);
+ List<ExtendedBlocklet> blocklets = blockletMap.prune(filteredSegment, null, partitions);
for (ExtendedBlocklet blocklet : blocklets) {
String blockName = blocklet.getPath();
blockName = CarbonTablePath.getCarbonDataFileName(blockName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
index 006ffd2..22a6e53 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
@@ -58,12 +58,13 @@ public class CarbonOutputMapperTest extends TestCase {
assert (file.exists());
File[] listFiles = file.listFiles(new FilenameFilter() {
@Override public boolean accept(File dir, String name) {
- return name.endsWith(".carbondata") || name.endsWith(".carbonindex");
+ return name.endsWith(".carbondata") ||
+ name.endsWith(".carbonindex") ||
+ name.endsWith(".carbonindexmerge");
}
});
assert (listFiles.length == 2);
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
new file mode 100644
index 0000000..9a9940b
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ test("show partitions on partition table") {
+ sql(
+ """
+ | CREATE TABLE partitionshow (designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int, empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionshow OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkExistence(sql(s"""SHOW PARTITIONS partitionshow"""), true, "empno=11", "empno=12")
+ }
+
+ test("droping on partition table for int partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(
+ sql(s"""select count (*) from partitionone"""),
+ sql(s"""select count (*) from originTable"""))
+
+ checkAnswer(
+ sql(s"""select count (*) from partitionone where empno=11"""),
+ sql(s"""select count (*) from originTable where empno=11"""))
+
+ sql(s"""ALTER TABLE partitionone DROP PARTITION(empno='11')""")
+
+ checkExistence(sql(s"""SHOW PARTITIONS partitionone"""), false, "empno=11")
+
+ checkAnswer(
+ sql(s"""select count (*) from partitionone where empno=11"""),
+ Seq(Row(0)))
+ }
+
+ test("dropping partition on table for more partition columns") {
+ sql(
+ """
+ | CREATE TABLE partitionmany (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""ALTER TABLE partitionmany DROP PARTITION(deptname='Learning')""")
+ checkExistence(sql(s"""SHOW PARTITIONS partitionmany"""), false, "deptname=Learning", "projectcode=928479")
+ checkAnswer(
+ sql(s"""select count (*) from partitionmany where deptname='Learning'"""),
+ Seq(Row(0)))
+ }
+
+ test("dropping all partition on table") {
+ sql(
+ """
+ | CREATE TABLE partitionall (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionall OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionall OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='Learning')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='configManagement')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='network')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='protocol')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='security')""")
+ assert(sql(s"""SHOW PARTITIONS partitionall""").collect().length == 0)
+ checkAnswer(
+ sql(s"""select count (*) from partitionall"""),
+ Seq(Row(0)))
+ }
+
+ test("dropping static partition on table") {
+ sql(
+ """
+ | CREATE TABLE staticpartition (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int,workgroupcategory int, empname String, designation String)
+ | PARTITIONED BY (deptname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into staticpartition PARTITION(deptname='software') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+
+ checkExistence(sql(s"""SHOW PARTITIONS staticpartition"""), true, "deptname=software")
+ assert(sql(s"""SHOW PARTITIONS staticpartition""").collect().length == 1)
+ sql(s"""ALTER TABLE staticpartition DROP PARTITION(deptname='software')""")
+ checkAnswer(
+ sql(s"""select count (*) from staticpartition"""),
+ Seq(Row(0)))
+ sql(s"""insert into staticpartition select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation,deptname from originTable""")
+ checkExistence(sql(s"""SHOW PARTITIONS staticpartition"""), true, "deptname=protocol")
+ checkAnswer(
+ sql(s"""select count (*) from staticpartition"""),
+ sql(s"""select count (*) from originTable"""))
+
+ }
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitionall")
+ sql("drop table if exists partitionmany")
+ sql("drop table if exists partitionshow")
+ sql("drop table if exists staticpartition")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
new file mode 100644
index 0000000..09d9da1
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.rdd
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.{Partition, SparkContext, TaskContext}
+
+import org.apache.carbondata.core.metadata.PartitionMapFileStore
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+case class CarbonDropPartition(rddId: Int, val idx: Int, segmentPath: String)
+ extends Partition {
+
+ override val index: Int = idx
+
+ override def hashCode(): Int = 41 * (41 + rddId) + idx
+}
+
+/**
+ * RDD to drop the partitions from partition mapper files of all segments.
+ * @param sc
+ * @param tablePath
+ * @param segments segments to be merged
+ */
+class CarbonDropPartitionRDD(
+ sc: SparkContext,
+ tablePath: String,
+ segments: Seq[String],
+ partitions: Seq[String],
+ uniqueId: String)
+ extends CarbonRDD[String](sc, Nil) {
+
+ override def getPartitions: Array[Partition] = {
+ segments.zipWithIndex.map {s =>
+ CarbonDropPartition(id, s._2, CarbonTablePath.getSegmentPath(tablePath, s._1))
+ }.toArray
+ }
+
+ override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[String] = {
+ val iter = new Iterator[String] {
+ val split = theSplit.asInstanceOf[CarbonDropPartition]
+ logInfo("Dropping partition information from : " + split.segmentPath)
+
+ new PartitionMapFileStore().dropPartitions(
+ split.segmentPath,
+ partitions.toList.asJava,
+ uniqueId)
+
+ var havePair = false
+ var finished = false
+
+ override def hasNext: Boolean = {
+ if (!finished && !havePair) {
+ finished = true
+ havePair = !finished
+ }
+ !finished
+ }
+
+ override def next(): String = {
+ if (!hasNext) {
+ throw new java.util.NoSuchElementException("End of stream")
+ }
+ havePair = false
+ ""
+ }
+
+ }
+ iter
+ }
+
+}
+
+/**
+ * This RDD is used for committing the partitions which were removed in before step. It just removes
+ * old mapper files and related data files.
+ * @param sc
+ * @param tablePath
+ * @param segments segments to be merged
+ */
+class CarbonDropPartitionRollbackRDD(
+ sc: SparkContext,
+ tablePath: String,
+ segments: Seq[String],
+ uniqueId: String)
+ extends CarbonRDD[String](sc, Nil) {
+
+ override def getPartitions: Array[Partition] = {
+ segments.zipWithIndex.map {s =>
+ CarbonDropPartition(id, s._2, CarbonTablePath.getSegmentPath(tablePath, s._1))
+ }.toArray
+ }
+
+ override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[String] = {
+ val iter = new Iterator[String] {
+ val split = theSplit.asInstanceOf[CarbonDropPartition]
+ logInfo("Commit partition information from : " + split.segmentPath)
+
+ new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, false)
+
+ var havePair = false
+ var finished = false
+
+ override def hasNext: Boolean = {
+ if (!finished && !havePair) {
+ finished = true
+ havePair = !finished
+ }
+ !finished
+ }
+
+ override def next(): String = {
+ if (!hasNext) {
+ throw new java.util.NoSuchElementException("End of stream")
+ }
+ havePair = false
+ ""
+ }
+
+ }
+ iter
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
index 0f745cc..833c6fe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql
+import scala.collection.JavaConverters._
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapred.JobConf
@@ -24,9 +26,10 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
@@ -36,6 +39,7 @@ import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
case class CarbonCountStar(
attributesRaw: Seq[Attribute],
carbonTable: CarbonTable,
+ sparkSession: SparkSession,
outUnsafeRows: Boolean = true) extends LeafExecNode {
override def doExecute(): RDD[InternalRow] = {
@@ -45,7 +49,13 @@ case class CarbonCountStar(
// get row count
val rowCount = CarbonUpdateUtil.getRowCount(
- tableInputFormat.getBlockRowCount(job, absoluteTableIdentifier),
+ tableInputFormat.getBlockRowCount(
+ job,
+ absoluteTableIdentifier,
+ CarbonFilters.getPartitions(
+ Seq.empty,
+ sparkSession,
+ TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName))).asJava),
absoluteTableIdentifier)
val value = new GenericInternalRow(Seq(Long.box(rowCount)).toArray.asInstanceOf[Array[Any]])
val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index b043698..d2e8789 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -29,7 +29,9 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.ExecutionErrors
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -92,8 +94,14 @@ object DeleteExecution {
if (keyRdd.partitions.length == 0) {
return true
}
-
- val blockMappingVO = carbonInputFormat.getBlockRowCount(job, absoluteTableIdentifier)
+ val blockMappingVO =
+ carbonInputFormat.getBlockRowCount(
+ job,
+ absoluteTableIdentifier,
+ CarbonFilters.getPartitions(
+ Seq.empty,
+ sparkSession,
+ TableIdentifier(tableName, databaseNameOp)).asJava)
val segmentUpdateStatusMngr = new SegmentUpdateStatusManager(absoluteTableIdentifier)
CarbonUpdateUtil
.createBlockDetailsMap(blockMappingVO, segmentUpdateStatusMngr)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
new file mode 100644
index 0000000..b787aa7
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.partition
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
+import org.apache.spark.util.AlterTableUtil
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.spark.rdd.{CarbonDropPartitionRDD, CarbonDropPartitionRollbackRDD}
+
+/**
+ * Drop the partitions from hive and carbon store. It drops the partitions in following steps
+ * 1. Drop the partitions from carbon store, it just create one new mapper file in each segment
+ * with uniqueid.
+ * 2. Drop partitions from hive.
+ * 3. In any above step fails then roll back the newly created files
+ * 4. After success of steps 1 and 2 , it commits the files by removing the old fails.
+ * Here it does not remove any data from store. During compaction the old data won't be considered.
+ * @param tableName
+ * @param specs
+ * @param ifExists
+ * @param purge
+ * @param retainData
+ */
+case class CarbonStandardAlterTableDropPartition(
+ tableName: TableIdentifier,
+ specs: Seq[TablePartitionSpec],
+ ifExists: Boolean,
+ purge: Boolean,
+ retainData: Boolean)
+ extends AtomicRunnableCommand {
+
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ if (table.isHivePartitionTable) {
+ try {
+ specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
+ } catch {
+ case e: Exception =>
+ if (!ifExists) {
+ throw e
+ } else {
+ log.warn(e.getMessage)
+ return Seq.empty[Row]
+ }
+ }
+
+ // Drop the partitions from hive.
+ AlterTableDropPartitionCommand(tableName, specs, ifExists, purge, retainData)
+ .run(sparkSession)
+ }
+ Seq.empty[Row]
+ }
+
+
+ override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
+ AlterTableAddPartitionCommand(tableName, specs.map((_, None)), ifExists)
+ val msg = s"Got exception $exception when processing data of drop partition." +
+ "Adding back partitions to the metadata"
+ LogServiceFactory.getLogService(this.getClass.getCanonicalName).error(msg)
+ Seq.empty[Row]
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ var locks = List.empty[ICarbonLock]
+ val uniqueId = System.currentTimeMillis().toString
+ try {
+ val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
+ LockUsage.COMPACTION_LOCK,
+ LockUsage.DELETE_SEGMENT_LOCK,
+ LockUsage.DROP_TABLE_LOCK,
+ LockUsage.CLEAN_FILES_LOCK,
+ LockUsage.ALTER_PARTITION_LOCK)
+ locks = AlterTableUtil.validateTableAndAcquireLock(
+ table.getDatabaseName,
+ table.getTableName,
+ locksToBeAcquired)(sparkSession)
+ val partitionNames = specs.flatMap { f =>
+ f.map(k => k._1 + "=" + k._2)
+ }.toSet
+ val segments = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
+ .getValidAndInvalidSegments.getValidSegments
+ try {
+ // First drop the partitions from partition mapper files of each segment
+ new CarbonDropPartitionRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ partitionNames.toSeq,
+ uniqueId).collect()
+ } catch {
+ case e: Exception =>
+ // roll back the drop partitions from carbon store
+ new CarbonDropPartitionRollbackRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ uniqueId).collect()
+ throw e
+ }
+ val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
+ .getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
+ CarbonUpdateUtil
+ .updateTableMetadataStatus(segmentSet,
+ table,
+ uniqueId,
+ true,
+ new util.ArrayList[String])
+ DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
+ } finally {
+ AlterTableUtil.releaseLocks(locks)
+ }
+ Seq.empty[Row]
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index f86b35f..dac08e0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -98,11 +98,11 @@ case class CarbonCreateTableCommand(
val partitionString =
if (partitionInfo != null &&
partitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
- s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
- _.getColumnName).mkString(",")})"
- } else {
- ""
- }
+ s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
+ _.getColumnName).mkString(",")})"
+ } else {
+ ""
+ }
sparkSession.sql(
s"""CREATE TABLE $dbName.$tableName
|(${ rawSchema })
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 1e71714..ad519e6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -79,7 +79,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
case CountStarPlan(colAttr, PhysicalOperation(projectList, predicates, l: LogicalRelation))
if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] && driverSideCountStar(l) =>
val relation = l.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- CarbonCountStar(colAttr, relation.carbonTable) :: Nil
+ CarbonCountStar(colAttr, relation.carbonTable, SparkSession.getActiveSession.get) :: Nil
case _ => Nil
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 21b81ce..eeadbf6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -23,12 +23,13 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompactionCommand, CarbonInsertIntoCommand, CarbonLoadDataCommand, RefreshCarbonTableCommand}
-import org.apache.spark.sql.execution.command.partition.CarbonShowCarbonPartitionsCommand
+import org.apache.spark.sql.execution.command.partition.{CarbonShowCarbonPartitionsCommand, CarbonStandardAlterTableDropPartition}
import org.apache.spark.sql.execution.command.schema._
import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedCommand, CarbonDropTableCommand}
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand}
import org.apache.spark.sql.CarbonExpressions.{CarbonDescribeTable => DescribeTableCommand}
import org.apache.spark.sql.execution.datasources.RefreshTable
+import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
@@ -158,10 +159,30 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(t)(sparkSession)
if (isCarbonTable) {
- ExecutedCommandExec(CarbonShowCarbonPartitionsCommand(t)) :: Nil
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(t)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
+ if (!carbonTable.isHivePartitionTable) {
+ ExecutedCommandExec(CarbonShowCarbonPartitionsCommand(t)) :: Nil
+ } else {
+ ExecutedCommandExec(ShowPartitionsCommand(t, cols)) :: Nil
+ }
} else {
ExecutedCommandExec(ShowPartitionsCommand(t, cols)) :: Nil
}
+ case adp@AlterTableDropPartitionCommand(tableName, specs, ifExists, purge, retainData) =>
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(tableName)(sparkSession)
+ if (isCarbonTable) {
+ ExecutedCommandExec(
+ CarbonStandardAlterTableDropPartition(
+ tableName,
+ specs,
+ ifExists,
+ purge,
+ retainData)) :: Nil
+ } else {
+ ExecutedCommandExec(adp) :: Nil
+ }
case set@SetCommand(kv) =>
ExecutedCommandExec(CarbonSetCommand(set)) :: Nil
case reset@ResetCommand =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ff55a2e/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
index 0cd1331..08d1497 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
@@ -70,7 +70,9 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
if (loadBatch.isLoading()) {
try {
loadBatch.readyRead();
- queue.put(loadBatch);
+ if (loadBatch.size > 0) {
+ queue.put(loadBatch);
+ }
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
[25/50] [abbrv] carbondata git commit: [CARBONDATA-1862][PARTITION]
Support compaction for partition table
Posted by ja...@apache.org.
[CARBONDATA-1862][PARTITION] Support compaction for partition table
It supports compaction on partition table.
There is a change in compaction during the block identification and grouping. As all blocks which are related same partition always needs to group to same set for compaction.So compactor needs to get the partition information from partition map file during compaction of partition table
This closes #1675
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5ed39de1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5ed39de1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5ed39de1
Branch: refs/heads/fgdatamap
Commit: 5ed39de193c0a589c30fbbd01e19e85715f2cac0
Parents: 3ff55a2
Author: ravipesala <ra...@gmail.com>
Authored: Sun Dec 17 11:09:34 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Dec 21 08:59:15 2017 +0800
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 4 +
.../core/metadata/schema/table/CarbonTable.java | 3 +-
...andardPartitionTableCompactionTestCase.scala | 178 +++++++++++++++++++
.../StandardPartitionTableQueryTestCase.scala | 4 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 72 ++++++--
.../spark/rdd/CarbonSparkPartition.scala | 3 +-
.../command/carbonTableSchemaCommon.scala | 13 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 15 +-
.../spark/rdd/CarbonTableCompactor.scala | 38 +++-
.../CarbonAlterTableCompactionCommand.scala | 9 +-
.../spark/sql/optimizer/CarbonFilters.scala | 13 ++
.../merger/CompactionResultSortProcessor.java | 29 ++-
.../merger/RowResultMergerProcessor.java | 19 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 3 +-
14 files changed, 353 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 8578cfe..f60b69f 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -276,6 +276,10 @@ public class PartitionMapFileStore {
return partitionMap.get(indexFileName);
}
+ public Map<String, List<String>> getPartitionMap() {
+ return partitionMap;
+ }
+
public static class PartitionMapper implements Serializable {
private static final long serialVersionUID = 3582245668420401089L;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 7732a50..8c4b08b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -571,7 +571,8 @@ public class CarbonTable implements Serializable {
}
public boolean isPartitionTable() {
- return null != tablePartitionMap.get(getTableName());
+ return null != tablePartitionMap.get(getTableName())
+ && tablePartitionMap.get(getTableName()).getPartitionType() != PartitionType.NATIVE_HIVE;
}
public boolean isHivePartitionTable() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
new file mode 100644
index 0000000..9056fea
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
+ carbonTable.getTablePath)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".partitionmap")
+ }
+ })
+ assert(dataFiles.length == partitions)
+ }
+
+ test("data compaction for partition table for one partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql("ALTER TABLE partitionone COMPACT 'MINOR'").collect()
+
+ validateDataFiles("default_partitionone", "0.1", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone where empno=11 order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where empno=11 order by empno"))
+
+ }
+
+
+ test("data compaction for partition table for three partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionthree (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql("ALTER TABLE partitionthree COMPACT 'MINOR'").collect()
+
+ validateDataFiles("default_partitionthree", "0.1", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionthree where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"))
+ }
+
+ test("data major compaction for partition table") {
+ sql(
+ """
+ | CREATE TABLE partitionmajor (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql("ALTER TABLE partitionmajor COMPACT 'MINOR'").collect()
+ sql(s"""ALTER TABLE partitionmajor DROP PARTITION(workgroupcategory='1')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmajor where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno").collect()
+ sql("ALTER TABLE partitionmajor COMPACT 'MAJOR'").collect()
+ validateDataFiles("default_partitionmajor", "0.2", 1)
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmajor where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"),
+ rows)
+ }
+
+ test("data compaction for static partition table") {
+ sql(
+ """
+ | CREATE TABLE staticpartition (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int,workgroupcategory int, empname String, designation String)
+ | PARTITIONED BY (deptname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into staticpartition PARTITION(deptname='software') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ sql(s"""insert into staticpartition PARTITION(deptname='software') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ sql(s"""insert into staticpartition PARTITION(deptname='finance') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ sql(s"""insert into staticpartition PARTITION(deptname='finance') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ val p1 = sql(s"""select count(*) from staticpartition where deptname='software'""").collect()
+ val p2 = sql(s"""select count(*) from staticpartition where deptname='finance'""").collect()
+ sql("ALTER TABLE staticpartition COMPACT 'MINOR'").collect()
+
+ validateDataFiles("default_staticpartition", "0.1", 1)
+
+ checkAnswer(sql(s"""select count(*) from staticpartition where deptname='software'"""), p1)
+ checkAnswer(sql(s"""select count(*) from staticpartition where deptname='finance'"""), p2)
+ }
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitiontwo")
+ sql("drop table if exists partitionthree")
+ sql("drop table if exists partitionmajor")
+ sql("drop table if exists staticpartition")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index 570951a..8670162 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -182,7 +182,7 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
"doj=2014-08-15 00:00:00",
"projectenddate=2016-12-30"))
checkAnswer(frame1,
- sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where doj>'2006-01-17 00:00:00'"))
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where doj>cast('2006-01-17 00:00:00' as Timestamp)"))
}
@@ -209,8 +209,6 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
sql("drop table if exists partitionmany")
sql("drop table if exists partitiondate")
sql("drop table if exists partitiondateinsert")
- sql("drop table if exists staticpartitionone")
- sql("drop table if exists singlepasspartitionone")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index fb4634e..59e5d30 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.mutate.UpdateVO
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, CarbonInputSplitTaskInfo}
@@ -85,6 +86,11 @@ class CarbonMergerRDD[K, V](
} else {
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
}
+ val partitionNames = if (carbonTable.isHivePartitionTable) {
+ carbonSparkPartition.partitionNames.get.asJava
+ } else {
+ null
+ }
// this property is used to determine whether temp location for carbon is inside
// container temp dir or is yarn application directory.
val carbonUseLocalDir = CarbonProperties.getInstance()
@@ -207,11 +213,13 @@ class CarbonMergerRDD[K, V](
carbonLoadModel.setPartitionId("0")
var processor: AbstractResultProcessor = null
if (restructuredBlockExists) {
- processor = new CompactionResultSortProcessor(carbonLoadModel, carbonTable,
+ processor = new CompactionResultSortProcessor(
+ carbonLoadModel,
+ carbonTable,
segmentProperties,
carbonMergerMapping.campactionType,
- factTableName
- )
+ factTableName,
+ partitionNames)
} else {
processor =
new RowResultMergerProcessor(
@@ -220,8 +228,8 @@ class CarbonMergerRDD[K, V](
segmentProperties,
tempStoreLoc,
carbonLoadModel,
- carbonMergerMapping.campactionType
- )
+ carbonMergerMapping.campactionType,
+ partitionNames)
}
mergeStatus = processor.execute(result2)
mergeResult = tableBlockInfoList.get(0).getSegmentId + ',' + mergeNumber
@@ -269,6 +277,9 @@ class CarbonMergerRDD[K, V](
val jobConf: JobConf = new JobConf(new Configuration)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
+ CarbonTableInputFormat.setPartitionsToPrune(
+ job.getConfiguration,
+ carbonMergerMapping.currentPartitions.asJava)
CarbonTableInputFormat.setTableInfo(job.getConfiguration,
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getTableInfo)
var updateDetails: UpdateVO = null
@@ -308,7 +319,7 @@ class CarbonMergerRDD[K, V](
.map(_.asInstanceOf[CarbonInputSplit])
.filter { split => FileFormat.COLUMNAR_V3.equals(split.getFileFormat) }.toList.asJava
- carbonInputSplits ++:= splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).filter(entry => {
+ carbonInputSplits ++:= splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).filter{ entry =>
val blockInfo = new TableBlockInfo(entry.getPath.toString,
entry.getStart, entry.getSegmentId,
entry.getLocations, entry.getLength, entry.getVersion,
@@ -318,7 +329,7 @@ class CarbonMergerRDD[K, V](
.isInvalidTableBlock(blockInfo.getSegmentId, blockInfo.getFilePath,
updateDetails, updateStatusManager)))) &&
FileFormat.COLUMNAR_V3.equals(entry.getFileFormat)
- })
+ }
}
// prepare the details required to extract the segment properties using last segment.
@@ -337,25 +348,25 @@ class CarbonMergerRDD[K, V](
}
val columnToCardinalityMap = new util.HashMap[java.lang.String, Integer]()
-
- carbonInputSplits.foreach(splits => {
- val taskNo = splits.taskId
+ val partitionTaskMap = new util.HashMap[util.List[String], String]()
+ carbonInputSplits.foreach { split =>
+ val taskNo = getTaskNo(split, partitionTaskMap)
var dataFileFooter: DataFileFooter = null
val splitList = taskIdMapping.get(taskNo)
noOfBlocks += 1
if (null == splitList) {
val splitTempList = new util.ArrayList[CarbonInputSplit]()
- splitTempList.add(splits)
+ splitTempList.add(split)
taskIdMapping.put(taskNo, splitTempList)
} else {
- splitList.add(splits)
+ splitList.add(split)
}
// Check the cardinality of each columns and set the highest.
try {
dataFileFooter = CarbonUtil.readMetadatFile(
- CarbonInputSplit.getTableBlockInfo(splits))
+ CarbonInputSplit.getTableBlockInfo(split))
} catch {
case e: IOException =>
logError("Exception in preparing the data file footer for compaction " + e.getMessage)
@@ -367,7 +378,6 @@ class CarbonMergerRDD[K, V](
dataFileFooter.getColumnInTable,
dataFileFooter.getSegmentInfo.getColumnCardinality)
}
- )
val updatedMaxSegmentColumnList = new util.ArrayList[ColumnSchema]()
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
// update cardinality and column schema list according to master schema
@@ -428,7 +438,12 @@ class CarbonMergerRDD[K, V](
carbonPartitionId = Integer.parseInt(taskInfo.getTaskId)
}
result.add(
- new CarbonSparkPartition(id, taskPartitionNo, multiBlockSplit, carbonPartitionId))
+ new CarbonSparkPartition(
+ id,
+ taskPartitionNo,
+ multiBlockSplit,
+ carbonPartitionId,
+ getPartitionNamesFromTask(taskInfo.getTaskId, partitionTaskMap)))
taskPartitionNo += 1
}
}
@@ -456,6 +471,33 @@ class CarbonMergerRDD[K, V](
result.toArray(new Array[Partition](result.size))
}
+ private def getTaskNo(
+ split: CarbonInputSplit,
+ partitionTaskMap: util.Map[List[String], String]): String = {
+ if (carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isHivePartitionTable) {
+ val partitions =
+ carbonMergerMapping.partitionMapper.getPartitionMap.get(
+ CarbonTablePath.getCarbonIndexFileName(split.getBlockPath))
+ var task = partitionTaskMap.get(partitions)
+ if (task == null) {
+ task = split.taskId
+ partitionTaskMap.put(partitions, task)
+ }
+ task
+ } else {
+ split.taskId
+ }
+ }
+
+ private def getPartitionNamesFromTask(taskId: String,
+ partitionTaskMap: util.Map[List[String], String]): Option[Seq[String]] = {
+ if (carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isHivePartitionTable) {
+ Some(partitionTaskMap.asScala.find(f => f._2.equals(taskId)).get._1.asScala)
+ } else {
+ None
+ }
+ }
+
override def getPreferredLocations(split: Partition): Seq[String] = {
val theSplit = split.asInstanceOf[CarbonSparkPartition]
theSplit.split.value.getLocations.filter(_ != "localhost")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
index cf539ba..036f1d1 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
@@ -25,7 +25,8 @@ class CarbonSparkPartition(
val rddId: Int,
val idx: Int,
@transient val multiBlockSplit: CarbonMultiBlockSplit,
- val partitionId: Int = 0)
+ val partitionId: Int = 0,
+ val partitionNames: Option[Seq[String]] = None)
extends Partition {
val split = new SerializableWritable[CarbonMultiBlockSplit](multiBlockSplit)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index ad6d876..c7a7b69 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -29,7 +29,8 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
+import org.apache.carbondata.core.metadata.PartitionMapFileStore.PartitionMapper
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema._
@@ -114,7 +115,9 @@ case class CarbonMergerMapping(
// maxSegmentColCardinality is Cardinality of last segment of compaction
var maxSegmentColCardinality: Array[Int],
// maxSegmentColumnSchemaList is list of column schema of last segment of compaction
- var maxSegmentColumnSchemaList: List[ColumnSchema])
+ var maxSegmentColumnSchemaList: List[ColumnSchema],
+ currentPartitions: Seq[String],
+ @transient partitionMapper: PartitionMapper)
case class NodeInfo(TaskId: String, noOfBlocks: Int)
@@ -134,13 +137,15 @@ case class UpdateTableModel(
case class CompactionModel(compactionSize: Long,
compactionType: CompactionType,
carbonTable: CarbonTable,
- isDDLTrigger: Boolean)
+ isDDLTrigger: Boolean,
+ currentPartitions: Seq[String])
case class CompactionCallableModel(carbonLoadModel: CarbonLoadModel,
carbonTable: CarbonTable,
loadsToMerge: util.List[LoadMetadataDetails],
sqlContext: SQLContext,
- compactionType: CompactionType)
+ compactionType: CompactionType,
+ currentPartitions: Seq[String])
case class AlterPartitionModel(carbonLoadModel: CarbonLoadModel,
segmentId: String,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 72c979a..8c9700c 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -40,6 +40,7 @@ import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, N
import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.sql.execution.command.{CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.constants.LoggerAction
@@ -209,11 +210,12 @@ object CarbonDataRDDFactory {
val compactionSize = CarbonDataMergerUtil.getCompactionSize(CompactionType.MAJOR)
- val newcompactionModel = CompactionModel(compactionSize,
+ val newcompactionModel = CompactionModel(
+ compactionSize,
compactionType,
table,
- compactionModel.isDDLTrigger
- )
+ compactionModel.isDDLTrigger,
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, table))
// proceed for compaction
try {
CompactionFactory.getCompactor(
@@ -724,11 +726,12 @@ object CarbonDataRDDFactory {
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
val compactionSize = 0
val isCompactionTriggerByDDl = false
- val compactionModel = CompactionModel(compactionSize,
+ val compactionModel = CompactionModel(
+ compactionSize,
CompactionType.MINOR,
carbonTable,
- isCompactionTriggerByDDl
- )
+ isCompactionTriggerByDDl,
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, carbonTable))
var storeLocation = ""
val configuredStore = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
if (null != configuredStore && configuredStore.nonEmpty) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 5f5a3d1..9768625 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -18,6 +18,7 @@
package org.apache.carbondata.spark.rdd
import java.util
+import java.util.{List, Map}
import java.util.concurrent.ExecutorService
import scala.collection.JavaConverters._
@@ -26,8 +27,11 @@ import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Futu
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.{CarbonMergerMapping, CompactionCallableModel, CompactionModel}
+import org.apache.carbondata.core.metadata.PartitionMapFileStore
+import org.apache.carbondata.core.metadata.PartitionMapFileStore.PartitionMapper
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableCompactionPostEvent, AlterTableCompactionPreEvent, AlterTableCompactionPreStatusUpdateEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
@@ -111,7 +115,8 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
compactionModel.carbonTable,
loadsToMerge,
sqlContext,
- compactionModel.compactionType)
+ compactionModel.compactionType,
+ compactionModel.currentPartitions)
triggerCompaction(compactionCallableModel)
}
@@ -121,6 +126,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
val sc = compactionCallableModel.sqlContext
val carbonLoadModel = compactionCallableModel.carbonLoadModel
val compactionType = compactionCallableModel.compactionType
+ val partitions = compactionCallableModel.currentPartitions
val tablePath = carbonLoadModel.getTablePath
val startTime = System.nanoTime()
val mergedLoadName = CarbonDataMergerUtil.getMergedLoadName(loadsToMerge)
@@ -130,7 +136,26 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
val validSegments: Array[String] = CarbonDataMergerUtil
.getValidSegments(loadsToMerge).split(',')
val mergeLoadStartTime = CarbonUpdateUtil.readCurrentTime()
- val carbonMergerMapping = CarbonMergerMapping(tablePath,
+ val partitionMapper = if (carbonTable.isHivePartitionTable) {
+ var partitionMap: util.Map[String, util.List[String]] = null
+ validSegments.foreach { segmentId =>
+ val localMapper = new PartitionMapFileStore()
+ localMapper.readAllPartitionsOfSegment(
+ CarbonTablePath.getSegmentPath(carbonLoadModel.getTablePath, segmentId))
+ if (partitionMap == null) {
+ partitionMap = localMapper.getPartitionMap
+ } else {
+ partitionMap.putAll(localMapper.getPartitionMap)
+ }
+ }
+ val mapper = new PartitionMapper()
+ mapper.setPartitionMap(partitionMap)
+ mapper
+ } else {
+ null
+ }
+ val carbonMergerMapping = CarbonMergerMapping(
+ tablePath,
carbonTable.getMetaDataFilepath,
mergedLoadName,
databaseName,
@@ -139,8 +164,9 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableId,
compactionType,
maxSegmentColCardinality = null,
- maxSegmentColumnSchemaList = null
- )
+ maxSegmentColumnSchemaList = null,
+ currentPartitions = partitions,
+ partitionMapper)
carbonLoadModel.setTablePath(carbonMergerMapping.hdfsStoreLocation)
carbonLoadModel.setLoadMetadataDetails(
SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath).toList.asJava)
@@ -196,7 +222,9 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
val mergedLoadNumber = CarbonDataMergerUtil.getLoadNumberFromLoadName(mergedLoadName)
CommonUtil.mergeIndexFiles(
sc.sparkContext, Seq(mergedLoadNumber), tablePath, carbonTable, false)
-
+ new PartitionMapFileStore().mergePartitionMapFiles(
+ CarbonTablePath.getSegmentPath(tablePath, mergedLoadNumber),
+ carbonLoadModel.getFactTimeStamp + "")
// trigger event for compaction
val alterTableCompactionPreStatusUpdateEvent: AlterTableCompactionPreStatusUpdateEvent =
AlterTableCompactionPreStatusUpdateEvent(sc.sparkSession,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 930dd57..cb0ff2d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.execution.command.{AlterTableModel, CarbonMergerMapping, CompactionModel, DataCommand}
import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
@@ -159,7 +160,8 @@ case class CarbonAlterTableCompactionCommand(
val compactionModel = CompactionModel(compactionSize,
compactionType,
carbonTable,
- isCompactionTriggerByDDl
+ isCompactionTriggerByDDl,
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, carbonTable)
)
val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
@@ -209,8 +211,9 @@ case class CarbonAlterTableCompactionCommand(
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableId,
compactionType,
maxSegmentColCardinality = null,
- maxSegmentColumnSchemaList = null
- )
+ maxSegmentColumnSchemaList = null,
+ compactionModel.currentPartitions,
+ null)
// trigger event for merge index
val operationContext = new OperationContext
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index a63b358..24fd732 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
@@ -392,6 +393,18 @@ object CarbonFilters {
}
}
+ def getCurrentPartitions(sparkSession: SparkSession,
+ carbonTable: CarbonTable): Seq[String] = {
+ if (carbonTable.isHivePartitionTable) {
+ CarbonFilters.getPartitions(
+ Seq.empty,
+ sparkSession,
+ TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
+ } else {
+ Seq.empty
+ }
+ }
+
def getPartitions(partitionFilters: Seq[Expression],
sparkSession: SparkSession,
identifier: TableIdentifier): Seq[String] = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index d115a7a..eece8f2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.PartitionMapFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -34,6 +35,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
@@ -127,21 +129,19 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
*/
private SortIntermediateFileMerger intermediateFileMerger;
- /**
- * @param carbonLoadModel
- * @param carbonTable
- * @param segmentProperties
- * @param compactionType
- * @param tableName
- */
+ private List<String> partitionNames;
+
+
public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
- SegmentProperties segmentProperties, CompactionType compactionType, String tableName) {
+ SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
+ List<String> partitionNames) {
this.carbonLoadModel = carbonLoadModel;
this.carbonTable = carbonTable;
this.segmentProperties = segmentProperties;
this.segmentId = carbonLoadModel.getSegmentId();
this.compactionType = compactionType;
this.tableName = tableName;
+ this.partitionNames = partitionNames;
}
/**
@@ -168,6 +168,19 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
} catch (Exception e) {
LOGGER.error(e, "Compaction failed: " + e.getMessage());
} finally {
+ if (partitionNames != null) {
+ try {
+ new PartitionMapFileStore().writePartitionMapFile(
+ CarbonTablePath.getSegmentPath(
+ carbonLoadModel.getTablePath(),
+ carbonLoadModel.getSegmentId()),
+ carbonLoadModel.getTaskNo(),
+ partitionNames);
+ } catch (IOException e) {
+ LOGGER.error(e, "Compaction failed: " + e.getMessage());
+ isCompactionSuccess = false;
+ }
+ }
// clear temp files and folders created during compaction
deleteTempStoreLocation();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index fbe2c85..3d0700b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.processing.merger;
+import java.io.IOException;
import java.util.AbstractQueue;
import java.util.Comparator;
import java.util.List;
@@ -29,10 +30,12 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.PartitionMapFileStore;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.exception.SliceMergerException;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
@@ -47,6 +50,8 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
private CarbonFactHandler dataHandler;
private SegmentProperties segprop;
+ private CarbonLoadModel loadModel;
+ private List<String> partitionNames;
/**
* record holder heap
*/
@@ -57,8 +62,10 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
public RowResultMergerProcessor(String databaseName,
String tableName, SegmentProperties segProp, String[] tempStoreLocation,
- CarbonLoadModel loadModel, CompactionType compactionType) {
+ CarbonLoadModel loadModel, CompactionType compactionType, List<String> partitionNames) {
this.segprop = segProp;
+ this.partitionNames = partitionNames;
+ this.loadModel = loadModel;
CarbonDataProcessorUtil.createLocations(tempStoreLocation);
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
@@ -150,8 +157,14 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
if (isDataPresent) {
this.dataHandler.closeHandler();
}
- } catch (CarbonDataWriterException e) {
- LOGGER.error("Exception while closing the handler in compaction merger " + e.getMessage());
+ if (partitionNames != null) {
+ new PartitionMapFileStore().writePartitionMapFile(
+ CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId()),
+ loadModel.getTaskNo(),
+ partitionNames);
+ }
+ } catch (CarbonDataWriterException | IOException e) {
+ LOGGER.error(e,"Exception in compaction merger");
mergeStatus = false;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ed39de1/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 37aaea5..765a88b 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -177,7 +177,8 @@ class StreamHandoffRDD[K, V](
carbonTable,
segmentProperties,
CompactionType.STREAMING,
- carbonTable.getTableName
+ carbonTable.getTableName,
+ null
)
}
[29/50] [abbrv] carbondata git commit: [CARBONDATA-1258] CarbonData
should not allow loading Date Type values violating the boundary condition
("0001-01-01" through "9999-12-31")
Posted by ja...@apache.org.
[CARBONDATA-1258] CarbonData should not allow loading Date Type values violating the boundary condition ("0001-01-01" through "9999-12-31")
Spark date type, supporting "0001-01-01" through "9999-12-31". As carbon pushes the data to spark layer so can support beyond this range.
This closes #1126
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/425f820b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/425f820b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/425f820b
Branch: refs/heads/fgdatamap
Commit: 425f820bded52998e2da3bc8712a52a1544e7138
Parents: 71959da
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Mon Jul 3 15:48:40 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Dec 21 12:22:09 2017 +0530
----------------------------------------------------------------------
.../DateDirectDictionaryGenerator.java | 28 +++++++
.../DateDirectDictionaryGeneratorTest.java | 84 ++++++++++++++++++++
2 files changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index cd32e97..c49af9c 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -43,11 +43,33 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private String dateFormat;
/**
+ * min value supported for date type column
+ */
+ private static final long MIN_VALUE;
+ /**
+ * MAx value supported for date type column
+ */
+ private static final long MAX_VALUE;
+ /**
* Logger instance
*/
private static final LogService LOGGER =
LogServiceFactory.getLogService(DateDirectDictionaryGenerator.class.getName());
+ static {
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
+ df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long minValue = 0;
+ long maxValue = 0;
+ try {
+ minValue = df.parse("0001-01-01").getTime();
+ maxValue = df.parse("9999-12-31").getTime();
+ } catch (ParseException e) {
+ // the Exception will not occur as constant value is being parsed
+ }
+ MIN_VALUE = minValue;
+ MAX_VALUE = maxValue;
+ }
public DateDirectDictionaryGenerator(String dateFormat) {
this.dateFormat = dateFormat;
initialize();
@@ -142,6 +164,12 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
}
private int generateKey(long timeValue) {
+ if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Value for date type column is not in valid range. Value considered as null.");
+ }
+ return 1;
+ }
return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
new file mode 100644
index 0000000..7390556
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.keygenerator.directdictionary;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test case for the DateDirectDictionaryGenerator
+ */
+public class DateDirectDictionaryGeneratorTest {
+ private DirectDictionaryGenerator dictionaryGenerator = null;
+
+ @Before public void setUp() throws Exception {
+ TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator();
+ dictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(DataTypes.DATE,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-01");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryInvalidValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-00");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("9999-12-31");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryInvalidValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("10000-12-31");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ @After public void tearDown() throws Exception {
+ dictionaryGenerator = null;
+ }
+}
[02/50] [abbrv] carbondata git commit: [CARBONDATA-1822] Added SDV
test cases for Refresh table command
Posted by ja...@apache.org.
[CARBONDATA-1822] Added SDV test cases for Refresh table command
[CARBONDATA-1822] Added SDV test cases for Refresh table command
This closes #1659
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8348752a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8348752a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8348752a
Branch: refs/heads/fgdatamap
Commit: 8348752a5fe9ea8fc7a9cb128ea95f5041f6f6b6
Parents: 40c7e8e
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Thu Dec 14 15:55:36 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Dec 16 23:35:53 2017 +0800
----------------------------------------------------------------------
.../sdv/register/TestRegisterCarbonTable.scala | 275 +++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 4 +-
2 files changed, 278 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8348752a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
new file mode 100644
index 0000000..45e3e50
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.cluster.sdv.register
+
+import java.io.IOException
+
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.hadoop.fs.{FileUtil, Path}
+import org.apache.spark.sql.test.TestQueryExecutor
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{AnalysisException, Row}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+
+/**
+ *
+ */
+class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
+ var dbLocationCustom = TestQueryExecutor.warehouse +
+ CarbonCommonConstants.FILE_SEPARATOR + "dbName"
+ override def beforeAll {
+ sql("drop database if exists carbon cascade")
+ }
+
+ def restoreData(dbLocationCustom: String, tableName: String) = {
+ val destination = dbLocationCustom + CarbonCommonConstants.FILE_SEPARATOR + tableName
+ val source = dbLocationCustom+ "_back" + CarbonCommonConstants.FILE_SEPARATOR + tableName
+ try {
+ val fs = new Path(source).getFileSystem(FileFactory.getConfiguration)
+ val sourceFileStatus = fs.getFileStatus(new Path(source))
+ FileUtil.copy(fs,
+ sourceFileStatus,
+ fs,
+ new Path(destination),
+ true,
+ true,
+ FileFactory.getConfiguration)
+ } catch {
+ case e : Exception =>
+ throw new IOException("carbon table data restore failed.")
+ } finally {
+
+ }
+ }
+ def backUpData(dbLocationCustom: String, tableName: String) = {
+ val source = dbLocationCustom + CarbonCommonConstants.FILE_SEPARATOR + tableName
+ val destination = dbLocationCustom+ "_back" + CarbonCommonConstants.FILE_SEPARATOR + tableName
+ try {
+ val fs = new Path(source).getFileSystem(FileFactory.getConfiguration)
+ val sourceFileStatus = fs.getFileStatus(new Path(source))
+ FileUtil.copy(fs,
+ sourceFileStatus,
+ fs,
+ new Path(destination),
+ false,
+ true,
+ FileFactory.getConfiguration)
+ } catch {
+ case e : Exception =>
+ throw new IOException("carbon table data backup failed.")
+ }
+ }
+
+ test("register tables test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ checkAnswer(sql("select count(*) from carbontable"), Row(1))
+ checkAnswer(sql("select c1 from carbontable"), Seq(Row("a")))
+ }
+
+ test("register table test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ checkAnswer(sql("select count(*) from carbontable"), Row(1))
+ checkAnswer(sql("select c1 from carbontable"), Seq(Row("a")))
+ }
+
+ test("register pre aggregate tables test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'aa','aaa'")
+ sql("insert into carbontable select 'a',10,'aa','aaa'")
+ sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
+ backUpData(dbLocationCustom, "carbontable")
+ backUpData(dbLocationCustom, "carbontable_preagg1")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ restoreData(dbLocationCustom, "carbontable_preagg1")
+ sql("refresh table carbontable")
+ checkAnswer(sql("select count(*) from carbontable"), Row(3))
+ checkAnswer(sql("select c1 from carbontable"), Seq(Row("a"), Row("b"), Row("a")))
+ checkAnswer(sql("select count(*) from carbontable_preagg1"), Row(2))
+ checkAnswer(sql("select carbontable_c1 from carbontable_preagg1"), Seq(Row("a"), Row("b")))
+ }
+
+ test("register pre aggregate table test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'aa','aaa'")
+ sql("insert into carbontable select 'a',10,'aa','aaa'")
+ sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
+ backUpData(dbLocationCustom, "carbontable")
+ backUpData(dbLocationCustom, "carbontable_preagg1")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ restoreData(dbLocationCustom, "carbontable_preagg1")
+ sql("refresh table carbontable")
+ checkAnswer(sql("select count(*) from carbontable"), Row(3))
+ checkAnswer(sql("select c1 from carbontable"), Seq(Row("a"), Row("b"), Row("a")))
+ checkAnswer(sql("select count(*) from carbontable_preagg1"), Row(2))
+ checkAnswer(sql("select carbontable_c1 from carbontable_preagg1"), Seq(Row("a"), Row("b")))
+ }
+
+ test("register pre aggregate table should fail if the aggregate table not copied") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'aa','aaa'")
+ sql("insert into carbontable select 'a',10,'aa','aaa'")
+ sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
+ backUpData(dbLocationCustom, "carbontable")
+ backUpData(dbLocationCustom, "carbontable_preagg1")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ try {
+ sql("refresh table carbontable")
+ assert(false)
+ } catch {
+ case e : AnalysisException =>
+ assert(true)
+ }
+ restoreData(dbLocationCustom, "carbontable_preagg1")
+ }
+
+ test("Update operation on carbon table should pass after registration or refresh") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'bb','bbb'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ // update operation
+ sql("""update carbon.carbontable d set (d.c2) = (d.c2 + 1) where d.c1 = 'a'""").show()
+ sql("""update carbon.carbontable d set (d.c2) = (d.c2 + 1) where d.c1 = 'b'""").show()
+ checkAnswer(
+ sql("""select c1,c2,c3,c5 from carbon.carbontable"""),
+ Seq(Row("a",2,"aa","aaa"),Row("b",2,"bb","bbb"))
+ )
+ }
+
+ test("Delete operation on carbon table") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'bb','bbb'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ // delete operation
+ sql("""delete from carbontable where c3 = 'aa'""").show
+ checkAnswer(
+ sql("""select c1,c2,c3,c5 from carbon.carbontable"""),
+ Seq(Row("b",1,"bb","bbb"))
+ )
+ sql("drop table carbontable")
+ }
+
+ test("Alter table add column test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'bb','bbb'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ sql("Alter table carbontable add columns(c4 string) " +
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='c4', 'DEFAULT.VALUE.c4'='def')")
+ checkAnswer(
+ sql("""select c1,c2,c3,c5,c4 from carbon.carbontable"""),
+ Seq(Row("a",1,"aa","aaa","def"), Row("b",1,"bb","bbb","def"))
+ )
+ sql("drop table carbontable")
+ }
+
+ test("Alter table change column datatype test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'bb','bbb'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ sql("Alter table carbontable change c2 c2 long")
+ checkAnswer(
+ sql("""select c1,c2,c3,c5 from carbon.carbontable"""),
+ Seq(Row("a",1,"aa","aaa"), Row("b",1,"bb","bbb"))
+ )
+ sql("drop table carbontable")
+ }
+
+ test("Alter table drop column test") {
+ sql("drop database if exists carbon cascade")
+ sql(s"create database carbon location '$dbLocationCustom'")
+ sql("use carbon")
+ sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
+ sql("insert into carbontable select 'a',1,'aa','aaa'")
+ sql("insert into carbontable select 'b',1,'bb','bbb'")
+ backUpData(dbLocationCustom, "carbontable")
+ sql("drop table carbontable")
+ restoreData(dbLocationCustom, "carbontable")
+ sql("refresh table carbontable")
+ sql("Alter table carbontable drop columns(c2)")
+ checkAnswer(
+ sql("""select * from carbon.carbontable"""),
+ Seq(Row("a","aa","aaa"), Row("b","bb","bbb"))
+ )
+ sql("drop table carbontable")
+ }
+
+ override def afterAll {
+ sql("use default")
+ sql("drop database if exists carbon cascade")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8348752a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index 2e00b0c..440d333 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -20,6 +20,7 @@ import org.apache.spark.sql.test.TestQueryExecutor
import org.scalatest.{BeforeAndAfterAll, Suites}
import org.apache.carbondata.cluster.sdv.generated._
+import org.apache.carbondata.cluster.sdv.register.TestRegisterCarbonTable
/**
* Suite class for all tests.
@@ -133,7 +134,8 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
new QueriesRangeFilterTestCase ::
new QueriesSparkBlockDistTestCase ::
new DataLoadingV3TestCase ::
- new QueriesIncludeDictionaryTestCase :: Nil
+ new QueriesIncludeDictionaryTestCase ::
+ new TestRegisterCarbonTable :: Nil
override val nestedSuites = suites.toIndexedSeq
[10/50] [abbrv] carbondata git commit: [CARBONDATA-1856][PARTITION]
Support insert/load data for partition table
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
new file mode 100644
index 0000000..99ce7fa
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources
+
+import java.io.File
+import java.util
+
+import scala.collection.JavaConverters._
+import scala.util.Random
+
+import org.apache.hadoop.fs.{FileStatus, Path}
+import org.apache.hadoop.io.NullWritable
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+import org.apache.spark.SparkEnv
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.DataSourceRegister
+import org.apache.spark.sql.types.{DataType, StructType}
+
+import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.metadata.{CarbonMetadata, PartitionMapFileStore}
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.hadoop.api.{CarbonOutputCommitter, CarbonTableOutputFormat}
+import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWriter
+import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel
+import org.apache.carbondata.spark.util.{DataLoadingUtil, Util}
+
+class CarbonFileFormat
+ extends FileFormat
+ with DataSourceRegister
+ with Logging
+with Serializable {
+
+ override def shortName(): String = "carbondata"
+
+ override def inferSchema(sparkSession: SparkSession,
+ options: Map[String, String],
+ files: Seq[FileStatus]): Option[StructType] = {
+ None
+ }
+
+ override def prepareWrite(
+ sparkSession: SparkSession,
+ job: Job,
+ options: Map[String, String],
+ dataSchema: StructType): OutputWriterFactory = {
+ val conf = job.getConfiguration
+ conf.setClass(
+ SQLConf.OUTPUT_COMMITTER_CLASS.key,
+ classOf[CarbonOutputCommitter],
+ classOf[CarbonOutputCommitter])
+ conf.set("carbon.commit.protocol", "carbon.commit.protocol")
+ sparkSession.sessionState.conf.setConfString(
+ "spark.sql.sources.commitProtocolClass",
+ "org.apache.spark.sql.execution.datasources.CarbonSQLHadoopMapReduceCommitProtocol")
+
+ job.setOutputFormatClass(classOf[CarbonTableOutputFormat])
+ var table = CarbonEnv.getCarbonTable(
+ TableIdentifier(options("tableName"), options.get("dbName")))(sparkSession)
+ val model = new CarbonLoadModel
+ val carbonProperty = CarbonProperties.getInstance()
+ val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
+ val tableProperties = table.getTableInfo.getFactTable.getTableProperties
+ optionsFinal.put("sort_scope", tableProperties.asScala.getOrElse("sort_scope",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))))
+ val partitionStr =
+ table.getTableInfo.getFactTable.getPartitionInfo.getColumnSchemaList.asScala.map(
+ _.getColumnName.toLowerCase).mkString(",")
+ optionsFinal.put(
+ "fileheader",
+ dataSchema.fields.map(_.name.toLowerCase).mkString(",") + "," + partitionStr)
+ DataLoadingUtil.buildCarbonLoadModel(
+ table,
+ carbonProperty,
+ options,
+ optionsFinal,
+ model,
+ conf
+ )
+ model.setPartitionId("0")
+ model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
+ model.setDictionaryServerHost(options.getOrElse("dicthost", null))
+ model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
+ CarbonTableOutputFormat.setLoadModel(conf, model)
+ CarbonTableOutputFormat.setOverwrite(conf, options("overwrite").toBoolean)
+
+ new OutputWriterFactory {
+
+ override def newInstance(
+ path: String,
+ dataSchema: StructType,
+ context: TaskAttemptContext): OutputWriter = {
+ val isCarbonUseMultiDir = CarbonProperties.getInstance().isUseMultiTempDir
+ var storeLocation: Array[String] = Array[String]()
+ val isCarbonUseLocalDir = CarbonProperties.getInstance()
+ .getProperty("carbon.use.local.dir", "false").equalsIgnoreCase("true")
+ val tmpLocationSuffix = File.separator + System.nanoTime()
+ if (isCarbonUseLocalDir) {
+ val yarnStoreLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (!isCarbonUseMultiDir && null != yarnStoreLocations && yarnStoreLocations.nonEmpty) {
+ // use single dir
+ storeLocation = storeLocation :+
+ (yarnStoreLocations(Random.nextInt(yarnStoreLocations.length)) + tmpLocationSuffix)
+ if (storeLocation == null || storeLocation.isEmpty) {
+ storeLocation = storeLocation :+
+ (System.getProperty("java.io.tmpdir") + tmpLocationSuffix)
+ }
+ } else {
+ // use all the yarn dirs
+ storeLocation = yarnStoreLocations.map(_ + tmpLocationSuffix)
+ }
+ } else {
+ storeLocation =
+ storeLocation :+ (System.getProperty("java.io.tmpdir") + tmpLocationSuffix)
+ }
+ CarbonTableOutputFormat.setTempStoreLocations(context.getConfiguration, storeLocation)
+ new CarbonOutputWriter(path, context, dataSchema.map(_.dataType))
+ }
+
+ override def getFileExtension(context: TaskAttemptContext): String = {
+ ".carbondata"
+ }
+
+ }
+ }
+}
+
+case class CarbonSQLHadoopMapReduceCommitProtocol(jobId: String, path: String, isAppend: Boolean)
+ extends SQLHadoopMapReduceCommitProtocol(jobId, path, isAppend) {
+ override def newTaskTempFileAbsPath(taskContext: TaskAttemptContext,
+ absoluteDir: String,
+ ext: String): String = {
+ val carbonFlow = taskContext.getConfiguration.get("carbon.commit.protocol")
+ if (carbonFlow != null) {
+ super.newTaskTempFile(taskContext, Some(absoluteDir), ext)
+ } else {
+ super.newTaskTempFileAbsPath(taskContext, absoluteDir, ext)
+ }
+ }
+}
+
+/**
+ * It is a just class to make compile between spark 2.1 and 2.2
+ */
+private trait AbstractCarbonOutputWriter {
+ def write(row: Row): Unit = throw new UnsupportedOperationException("call writeInternal")
+ def writeInternal(row: InternalRow): Unit = {
+ writeCarbon(row)
+ }
+ def write(row: InternalRow): Unit = {
+ writeCarbon(row)
+ }
+ def writeCarbon(row: InternalRow): Unit
+}
+
+private class CarbonOutputWriter(path: String,
+ context: TaskAttemptContext,
+ fieldTypes: Seq[DataType])
+ extends OutputWriter with AbstractCarbonOutputWriter {
+ val partitions = getPartitionsFromPath(path, context).map(ExternalCatalogUtils.unescapePathName)
+ val partitionData = if (partitions.nonEmpty) {
+ partitions.map(_.split("=")(1))
+ } else {
+ Array.empty
+ }
+ val writable = new StringArrayWritable()
+
+ private val recordWriter: CarbonRecordWriter = {
+
+ new CarbonTableOutputFormat() {
+ override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
+ new Path(path)
+ }
+ }.getRecordWriter(context).asInstanceOf[CarbonRecordWriter]
+ }
+
+ // TODO Implement writesupport interface to support writing Row directly to recordwriter
+ def writeCarbon(row: InternalRow): Unit = {
+ val data = new Array[String](fieldTypes.length + partitionData.length)
+ var i = 0
+ while (i < fieldTypes.length) {
+ if (!row.isNullAt(i)) {
+ data(i) = row.getString(i)
+ }
+ i += 1
+ }
+ if (partitionData.length > 0) {
+ System.arraycopy(partitionData, 0, data, fieldTypes.length, partitionData.length)
+ }
+ writable.set(data)
+ recordWriter.write(NullWritable.get(), writable)
+ }
+
+
+ override def writeInternal(row: InternalRow): Unit = {
+ writeCarbon(row)
+ }
+
+ override def close(): Unit = {
+ recordWriter.close(context)
+ val loadModel = recordWriter.getLoadModel
+ val segmentPath = CarbonTablePath.getSegmentPath(loadModel.getTablePath, loadModel.getSegmentId)
+ // write partition info to new file.
+ val partitonList = new util.ArrayList[String]()
+ partitions.foreach(partitonList.add)
+ new PartitionMapFileStore().writePartitionMapFile(
+ segmentPath,
+ loadModel.getTaskNo,
+ partitonList)
+ }
+
+ def getPartitionsFromPath(path: String, attemptContext: TaskAttemptContext): Array[String] = {
+ var attemptId = attemptContext.getTaskAttemptID.toString + "/"
+ if (path.indexOf(attemptId) <= 0) {
+ val model = CarbonTableOutputFormat.getLoadModel(attemptContext.getConfiguration)
+ attemptId = model.getTableName + "/"
+ }
+ val str = path.substring(path.indexOf(attemptId) + attemptId.length, path.lastIndexOf("/"))
+ if (str.length > 0) {
+ str.split("/")
+ } else {
+ Array.empty
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 8b247f7..7f9bdf7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -40,7 +40,6 @@ import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.format.DataType
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.spark.util.CarbonScalaUtil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 8745900..21b81ce 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -74,8 +74,8 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
CarbonDropTableCommand(ifNotExists, identifier.database,
identifier.table.toLowerCase)) :: Nil
case InsertIntoCarbonTable(relation: CarbonDatasourceHadoopRelation,
- _, child: LogicalPlan, overwrite, _) =>
- ExecutedCommandExec(CarbonInsertIntoCommand(relation, child, overwrite)) :: Nil
+ partition, child: LogicalPlan, overwrite, _) =>
+ ExecutedCommandExec(CarbonInsertIntoCommand(relation, child, overwrite, partition)) :: Nil
case createDb@CreateDatabaseCommand(dbName, ifNotExists, _, _, _) =>
FileUtils.createDatabaseDirectory(dbName, CarbonProperties.getStorePath)
ExecutedCommandExec(createDb) :: Nil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
index de5fa7e..ff7e06a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
@@ -17,18 +17,25 @@
package org.apache.spark.sql.hive
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
import org.apache.spark.SPARK_VERSION
import org.apache.spark.sql._
import org.apache.spark.sql.CarbonExpressions.CarbonUnresolvedRelation
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar}
-import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, NamedExpression}
import org.apache.spark.sql.catalyst.plans.Inner
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.execution.command.mutation.CarbonProjectForDeleteCommand
+import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, FileFormat, HadoopFsRelation, LogicalRelation}
+import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+
case class CarbonIUDAnalysisRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
private lazy val parser = sparkSession.sessionState.sqlParser
@@ -200,3 +207,63 @@ case class CarbonIUDAnalysisRule(sparkSession: SparkSession) extends Rule[Logica
}
}
}
+
+/**
+ * Insert into carbon table from other source
+ */
+case class CarbonPreInsertionCasts(sparkSession: SparkSession) extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = {
+ plan.transform {
+ // Wait until children are resolved.
+ case p: LogicalPlan if !p.childrenResolved => p
+
+ case p@InsertIntoTable(relation: LogicalRelation, _, child, _, _)
+ if relation.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ castChildOutput(p, relation, child)
+ }
+ }
+
+ def castChildOutput(p: InsertIntoTable,
+ relation: LogicalRelation,
+ child: LogicalPlan): LogicalPlan = {
+ val carbonDSRelation = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
+ if (carbonDSRelation.carbonRelation.output.size > CarbonCommonConstants
+ .DEFAULT_MAX_NUMBER_OF_COLUMNS) {
+ CarbonException.analysisException(
+ s"Maximum number of columns supported: " +
+ s"${CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS}")
+ }
+ if (child.output.size >= carbonDSRelation.carbonRelation.output.size ||
+ carbonDSRelation.carbonTable.isHivePartitionTable) {
+ val newChildOutput = child.output.zipWithIndex.map { columnWithIndex =>
+ columnWithIndex._1 match {
+ case attr: Alias =>
+ Alias(attr.child, s"col${ columnWithIndex._2 }")(attr.exprId)
+ case attr: Attribute =>
+ Alias(attr, s"col${ columnWithIndex._2 }")(NamedExpression.newExprId)
+ case attr => attr
+ }
+ }
+ val version = SPARK_VERSION
+ val newChild: LogicalPlan = if (newChildOutput == child.output) {
+ if (version.startsWith("2.1")) {
+ CarbonReflectionUtils.getField("child", p).asInstanceOf[LogicalPlan]
+ } else if (version.startsWith("2.2")) {
+ CarbonReflectionUtils.getField("query", p).asInstanceOf[LogicalPlan]
+ } else {
+ throw new UnsupportedOperationException(s"Spark version $version is not supported")
+ }
+ } else {
+ Project(newChildOutput, child)
+ }
+
+ val overwrite = CarbonReflectionUtils.getOverWriteOption("overwrite", p)
+
+ InsertIntoCarbonTable(carbonDSRelation, p.partition, newChild, overwrite, true)
+ } else {
+ CarbonException.analysisException(
+ "Cannot insert into target table because number of columns mismatch")
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 623d309..c28cc44 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -1322,59 +1322,3 @@ object CarbonPreAggregateDataLoadingRules extends Rule[LogicalPlan] {
}
}
}
-
-/**
- * Insert into carbon table from other source
- */
-case class CarbonPreInsertionCasts(sparkSession: SparkSession) extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = {
- plan.transform {
- // Wait until children are resolved.
- case p: LogicalPlan if !p.childrenResolved => p
-
- case p@InsertIntoTable(relation: LogicalRelation, _, child, _, _)
- if relation.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- castChildOutput(p, relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation], child)
- }
- }
-
- def castChildOutput(p: InsertIntoTable,
- relation: CarbonDatasourceHadoopRelation,
- child: LogicalPlan): LogicalPlan = {
- if (relation.carbonRelation.output.size > CarbonCommonConstants
- .DEFAULT_MAX_NUMBER_OF_COLUMNS) {
- CarbonException.analysisException("Maximum number of columns supported:" +
- s"${CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS}")
- }
- if (child.output.size >= relation.carbonRelation.output.size) {
- val newChildOutput = child.output.zipWithIndex.map { columnWithIndex =>
- columnWithIndex._1 match {
- case attr: Alias =>
- Alias(attr.child, s"col${ columnWithIndex._2 }")(attr.exprId)
- case attr: Attribute =>
- Alias(attr, s"col${ columnWithIndex._2 }")(NamedExpression.newExprId)
- case attr => attr
- }
- }
- val version = SPARK_VERSION
- val newChild: LogicalPlan = if (newChildOutput == child.output) {
- if (version.startsWith("2.1")) {
- CarbonReflectionUtils.getField("child", p).asInstanceOf[LogicalPlan]
- } else if (version.startsWith("2.2")) {
- CarbonReflectionUtils.getField("query", p).asInstanceOf[LogicalPlan]
- } else {
- throw new UnsupportedOperationException(s"Spark version $version is not supported")
- }
- } else {
- Project(newChildOutput, child)
- }
-
- val overwrite = CarbonReflectionUtils.getOverWriteOption("overwrite", p)
-
- InsertIntoCarbonTable(relation, p.partition, newChild, overwrite, true)
- } else {
- CarbonException.analysisException(
- "Cannot insert into target table because number of columns mismatch")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 55d784f..79095ca 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -199,7 +199,7 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
// validate partition clause
if (partitionFields.nonEmpty) {
if (!CommonUtil.validatePartitionColumns(tableProperties, partitionFields)) {
- throw new MalformedCarbonCommandException("Error: Invalid partition definition")
+ throw new MalformedCarbonCommandException("Error: Invalid partition definition")
}
// partition columns should not be part of the schema
val badPartCols = partitionFields.map(_.partitionColumn).toSet.intersect(colNames.toSet)
@@ -251,7 +251,7 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser)
}
if (partitionerFields.nonEmpty) {
if (!CommonUtil.validatePartitionColumns(tableProperties, partitionerFields)) {
- throw new MalformedCarbonCommandException("Error: Invalid partition definition")
+ throw new MalformedCarbonCommandException("Error: Invalid partition definition")
}
// partition columns should not be part of the schema
val badPartCols = partitionerFields.map(_.partitionColumn).toSet.intersect(colNames.toSet)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
index abe90f1..0cd1331 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
@@ -55,7 +55,7 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
@Override
public String[] next() {
- if (readBatch == null || !readBatch.hasNext()) {
+ if (readBatch == null || !readBatch.hasNext() && !close) {
try {
readBatch = queue.take();
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
index c32aa51..d3caa99 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
@@ -46,16 +46,16 @@ public abstract class AbstractResultProcessor {
CarbonFactDataHandlerModel carbonFactDataHandlerModel) {
CarbonDataFileAttributes carbonDataFileAttributes;
if (compactionType == CompactionType.IUD_UPDDEL_DELTA) {
- int taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegmentId(),
+ long taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegmentId(),
CarbonStorePath.getCarbonTablePath(loadModel.getTablePath(),
carbonTable.getCarbonTableIdentifier()));
// Increase the Task Index as in IUD_UPDDEL_DELTA_COMPACTION the new file will
// be written in same segment. So the TaskNo should be incremented by 1 from max val.
- int index = taskNo + 1;
+ long index = taskNo + 1;
carbonDataFileAttributes = new CarbonDataFileAttributes(index, loadModel.getFactTimeStamp());
} else {
carbonDataFileAttributes =
- new CarbonDataFileAttributes(Integer.parseInt(loadModel.getTaskNo()),
+ new CarbonDataFileAttributes(Long.parseLong(loadModel.getTaskNo()),
loadModel.getFactTimeStamp());
}
carbonFactDataHandlerModel.setCarbonDataFileAttributes(carbonDataFileAttributes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 48c5471..68a212e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -51,7 +51,7 @@ public class RowResultProcessor {
CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable,
segProp, tableName, tempStoreLocation);
CarbonDataFileAttributes carbonDataFileAttributes =
- new CarbonDataFileAttributes(Integer.parseInt(loadModel.getTaskNo()),
+ new CarbonDataFileAttributes(Long.parseLong(loadModel.getTaskNo()),
loadModel.getFactTimeStamp());
carbonFactDataHandlerModel.setCarbonDataFileAttributes(carbonDataFileAttributes);
carbonFactDataHandlerModel.setBucketId(bucketId);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
index b69815e..8bedd80 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
@@ -25,7 +25,7 @@ public class CarbonDataFileAttributes {
/**
* task Id which is unique for each spark task
*/
- private int taskId;
+ private long taskId;
/**
* load start time
@@ -36,7 +36,7 @@ public class CarbonDataFileAttributes {
* @param taskId
* @param factTimeStamp
*/
- public CarbonDataFileAttributes(int taskId, long factTimeStamp) {
+ public CarbonDataFileAttributes(long taskId, long factTimeStamp) {
this.taskId = taskId;
this.factTimeStamp = factTimeStamp;
}
@@ -44,7 +44,7 @@ public class CarbonDataFileAttributes {
/**
* @return
*/
- public int getTaskId() {
+ public long getTaskId() {
return taskId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index a8ae513..96bd2e3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -223,7 +223,7 @@ public class CarbonFactDataHandlerModel {
}
CarbonDataFileAttributes carbonDataFileAttributes =
- new CarbonDataFileAttributes(Integer.parseInt(configuration.getTaskNo()),
+ new CarbonDataFileAttributes(Long.parseLong(configuration.getTaskNo()),
(Long) configuration.getDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP));
String carbonDataDirectoryPath = getCarbonDataFolderLocation(configuration);
[36/50] [abbrv] carbondata git commit: [CARBONDATA-1923] Remove file
after running example/CompareTest.scala
Posted by ja...@apache.org.
[CARBONDATA-1923] Remove file after running example/CompareTest.scala
Remove file after running example/CompareTest.scala
This closes #1698
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e2360c74
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e2360c74
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e2360c74
Branch: refs/heads/fgdatamap
Commit: e2360c7499edbdb9615fa073c7f8ad29fb8cb1c2
Parents: f79b9ea
Author: xubo245 <60...@qq.com>
Authored: Wed Dec 20 21:10:59 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Dec 22 10:27:34 2017 +0800
----------------------------------------------------------------------
.../main/scala/org/apache/carbondata/examples/CompareTest.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2360c74/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index 414f8e1..fe87d7c 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.types._
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
/**
* A query test case
@@ -384,6 +384,8 @@ object CompareTest {
prepareTable(spark, table1, table2)
runTest(spark, table1, table2)
+ CarbonUtil.deleteFoldersAndFiles(new File(table1))
+ spark.sql(s"drop table if exists $table2")
spark.close()
}
[20/50] [abbrv] carbondata git commit: [CARBONDATA-1752] Remove
unused import
Posted by ja...@apache.org.
[CARBONDATA-1752] Remove unused import
Remove unused import
This closes #1691
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/15f04c35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/15f04c35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/15f04c35
Branch: refs/heads/fgdatamap
Commit: 15f04c35c24628f339b3e5917ef7b0e9d3751c22
Parents: a86c54b
Author: xubo245 <60...@qq.com>
Authored: Wed Dec 20 17:00:55 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Dec 20 17:56:31 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/CarbonPropertiesValidationTest.java | 1 -
.../apache/carbondata/core/cache/CacheProviderTest.java | 2 --
.../cache/dictionary/DictionaryCacheLoaderImplTest.java | 1 -
.../core/cache/dictionary/ForwardDictionaryCacheTest.java | 1 -
.../core/dictionary/client/DictionaryClientTest.java | 1 -
.../generator/ServerDictionaryGeneratorTest.java | 1 -
.../generator/TableDictionaryGeneratorTest.java | 1 -
.../mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java | 1 -
.../core/reader/CarbonDictionaryReaderImplTest.java | 1 -
.../CarbonDictionarySortIndexReaderImplTest.java | 1 -
.../expression/conditional/ListExpressionUnitTest.java | 1 -
.../core/scan/filter/FilterExpressionProcessorTest.java | 2 --
.../carbondata/core/util/CarbonMetadataUtilTest.java | 1 -
.../carbondata/core/util/DataFileFooterConverterTest.java | 1 -
.../org/apache/carbondata/core/util/DataTypeUtilTest.java | 1 -
.../CarbonDictionarySortIndexWriterImplTest.java | 1 -
.../carbondata/presto/readers/ObjectStreamReader.java | 1 -
.../carbondata/presto/readers/SliceStreamReader.java | 8 --------
.../testsuite/allqueries/MeasureOnlyTableTestCases.scala | 7 ++++---
.../CompactionSupportGlobalSortFunctionTest.scala | 3 ---
.../DataCompactionBlockletBoundryTest.scala | 4 +++-
.../testsuite/datacompaction/DataCompactionLockTest.scala | 3 ++-
.../datacompaction/MajorCompactionIgnoreInMinorTest.scala | 10 ++++------
.../MajorCompactionStopsAfterCompaction.scala | 7 +++----
.../apache/carbondata/lcm/locks/LocalFileLockTest.java | 1 -
.../apache/carbondata/lcm/locks/ZooKeeperLockingTest.java | 4 ----
26 files changed, 16 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
index 1e4569d..27a9a6d 100644
--- a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
@@ -22,7 +22,6 @@ import java.lang.reflect.Method;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonProperty;
import junit.framework.TestCase;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
index 38f5a7b..0617af3 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
@@ -28,11 +28,9 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.avro.Schema;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
index 83e9217..8bf8a07 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.format.ColumnDictionaryChunk;
import mockit.Mock;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
index 89fbf2d..d0aedd4 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
@@ -33,7 +33,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index 2fd69f8..dbd29e5 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -40,7 +40,6 @@ import mockit.Mock;
import mockit.MockUp;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
index 57bd3c0..95d73a9 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
import org.junit.After;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
index f535a59..1ea77f8 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
@@ -22,7 +22,6 @@ import java.util.Arrays;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
-import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java
index bc35093..31eb2a7 100644
--- a/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGeneratorUnitTest.java
@@ -21,7 +21,6 @@ import org.junit.Before;
import org.junit.Test;
import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNull;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
index 70f1740..d3c3bc3 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.service.impl.PathFactory;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import mockit.Mock;
import mockit.MockUp;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
index 68d34cf..f6fcff9 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
@@ -29,7 +29,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
index f826851..b885fc8 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
@@ -33,7 +33,6 @@ import java.util.ArrayList;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
public class ListExpressionUnitTest {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
index 1f244a3..14d4c1f 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
@@ -16,10 +16,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
-import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
import org.apache.carbondata.core.scan.expression.conditional.InExpression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 28cfcf2..40463a6 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.page.EncodedTablePage;
-import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.datastore.page.key.TablePageKey;
import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index 30b3247..de64c0a 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.reader.ThriftReader;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index bb528b9..1975d7a 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -21,7 +21,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
index f74a254..7ab202f 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
@@ -29,7 +29,6 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ObjectStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ObjectStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ObjectStreamReader.java
index d642e41..82d62ad 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ObjectStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/ObjectStreamReader.java
@@ -23,7 +23,6 @@ import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
import com.facebook.presto.spi.type.Type;
-import io.airlift.slice.Slice;
/**
* Class to read the Object Stream
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
index 36be07e..f307331 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
@@ -18,12 +18,6 @@
package org.apache.carbondata.presto.readers;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
@@ -31,8 +25,6 @@ import com.facebook.presto.spi.block.BlockBuilderStatus;
import com.facebook.presto.spi.block.DictionaryBlock;
import com.facebook.presto.spi.block.SliceArrayBlock;
import com.facebook.presto.spi.type.Type;
-import io.airlift.slice.Slice;
-import io.airlift.slice.Slices;
import static io.airlift.slice.Slices.utf8Slice;
import static io.airlift.slice.Slices.wrappedBuffer;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
index 4bc4bac..0a3869c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
@@ -18,13 +18,14 @@
package org.apache.carbondata.spark.testsuite.allqueries
import java.io.File
-import java.math.{BigDecimal, RoundingMode}
-import org.apache.spark.sql.{Row, SaveMode}
import org.scalatest.BeforeAndAfterAll
+
+import org.apache.spark.sql.SaveMode
+import org.apache.spark.sql.test.util.QueryTest
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.test.util.QueryTest
/**
* Test Class for all query on multiple datatypes
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
index 9014edb..8f891ce 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
@@ -17,8 +17,6 @@
package org.apache.carbondata.spark.testsuite.datacompaction
-import java.io.{File, FilenameFilter}
-
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
@@ -26,7 +24,6 @@ import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
class CompactionSupportGlobalSortFunctionTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
index 38367c9..11b6b93 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
@@ -17,9 +17,11 @@
package org.apache.carbondata.spark.testsuite.datacompaction
import org.scalatest.BeforeAndAfterAll
+
+import org.apache.spark.sql.test.util.QueryTest
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.test.util.QueryTest
/**
* FT for data compaction scenario.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
index 20a779d..2ef88a4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
@@ -22,13 +22,14 @@ import scala.collection.JavaConverters._
import org.scalatest.BeforeAndAfterAll
+import org.apache.spark.sql.test.util.QueryTest
+
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
-import org.apache.spark.sql.test.util.QueryTest
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 61de615..6155d7c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -20,18 +20,16 @@ import scala.collection.JavaConverters._
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper
-import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
+import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.hadoop.CacheClient
import org.apache.spark.sql.test.util.QueryTest
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-
/**
* FT for compaction scenario where major segment should not be included in minor.
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
index 7363838..bb1f829 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
@@ -20,13 +20,12 @@ import scala.collection.JavaConverters._
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
+import org.apache.spark.sql.test.util.QueryTest
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.test.util.QueryTest
-
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
/**
* FT for compaction scenario where major compaction will only compact the segments which are
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index ba77f29..812db5c 100644
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.lcm.locks;
import java.io.File;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.locks.LocalFileLock;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15f04c35/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
index e0b6d66..e7a0602 100644
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
+++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
@@ -16,11 +16,8 @@
*/
package org.apache.carbondata.lcm.locks;
-import mockit.NonStrictExpectations;
-
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.locks.ZooKeeperLocking;
@@ -32,7 +29,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
-import org.junit.Test;
import java.io.File;
import java.io.IOException;
[09/50] [abbrv] carbondata git commit: [CARBONDATA-1805][Dictionary]
Optimize pruning for dictionary loading
Posted by ja...@apache.org.
[CARBONDATA-1805][Dictionary] Optimize pruning for dictionary loading
Refactor the loadDataFrame method in GlobalDictionaryUtil, do not convert all columns to spark InternalRow, convert dictionary columns only instead.
This closes #1559
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6e224dce
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6e224dce
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6e224dce
Branch: refs/heads/fgdatamap
Commit: 6e224dce62c57c2b6c7a8df047663e64c6d77c4b
Parents: 4daf063
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Tue Nov 28 22:42:29 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Dec 18 16:24:49 2017 +0800
----------------------------------------------------------------------
.../spark/util/GlobalDictionaryUtil.scala | 95 ++++++++++++--------
1 file changed, 60 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e224dce/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index ee2c843..330582f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -37,7 +37,6 @@ import org.apache.spark.{Accumulator, SparkException}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.rdd.{NewHadoopRDD, RDD}
import org.apache.spark.sql._
-import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -346,36 +345,57 @@ object GlobalDictionaryUtil {
}
/**
- * load CSV files to DataFrame by using datasource "com.databricks.spark.csv"
+ * load and prune dictionary Rdd from csv file or input dataframe
*
- * @param sqlContext SQLContext
- * @param carbonLoadModel carbon data load model
+ * @param sqlContext sqlContext
+ * @param carbonLoadModel carbonLoadModel
+ * @param inputDF input dataframe
+ * @param requiredCols names of dictionary column
+ * @param hadoopConf hadoop configuration
+ * @return rdd that contains only dictionary columns
*/
- def loadDataFrame(sqlContext: SQLContext,
+ private def loadInputDataAsDictRdd(sqlContext: SQLContext,
carbonLoadModel: CarbonLoadModel,
- hadoopConf: Configuration): DataFrame = {
- CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
- hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
- val columnNames = carbonLoadModel.getCsvHeaderColumns
- val schema = StructType(columnNames.map[StructField, Array[StructField]] { column =>
- StructField(column, StringType)
- })
- val values = new Array[String](columnNames.length)
- val row = new StringArrayRow(values)
- val jobConf = new JobConf(hadoopConf)
- SparkHadoopUtil.get.addCredentials(jobConf)
- TokenCache.obtainTokensForNamenodes(jobConf.getCredentials,
- Array[Path](new Path(carbonLoadModel.getFactFilePath)),
- jobConf)
- val rdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
- sqlContext.sparkContext,
- classOf[CSVInputFormat],
- classOf[NullWritable],
- classOf[StringArrayWritable],
- jobConf).setName("global dictionary").map[Row] { currentRow =>
- row.setValues(currentRow._2.get())
+ inputDF: Option[DataFrame],
+ requiredCols: Array[String],
+ hadoopConf: Configuration): RDD[Row] = {
+ if (inputDF.isDefined) {
+ inputDF.get.select(requiredCols.head, requiredCols.tail : _*).rdd
+ } else {
+ CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
+ hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
+ val headerCols = carbonLoadModel.getCsvHeaderColumns.map(_.toLowerCase)
+ val header2Idx = headerCols.zipWithIndex.toMap
+ // index of dictionary columns in header
+ val dictColIdx = requiredCols.map(c => header2Idx(c.toLowerCase))
+
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ TokenCache.obtainTokensForNamenodes(jobConf.getCredentials,
+ Array[Path](new Path(carbonLoadModel.getFactFilePath)),
+ jobConf)
+ val dictRdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
+ sqlContext.sparkContext,
+ classOf[CSVInputFormat],
+ classOf[NullWritable],
+ classOf[StringArrayWritable],
+ jobConf)
+ .setName("global dictionary")
+ .map[Row] { currentRow =>
+ val rawRow = currentRow._2.get()
+ val destRow = new Array[String](dictColIdx.length)
+ for (i <- dictColIdx.indices) {
+ // dictionary index in this row
+ val idx = dictColIdx(i)
+ // copy specific dictionary value from source to dest
+ if (idx < rawRow.length) {
+ System.arraycopy(rawRow, idx, destRow, i, 1)
+ }
+ }
+ Row.fromSeq(destRow)
+ }
+ dictRdd
}
- sqlContext.createDataFrame(rdd, schema)
}
// Hack for spark2 integration
@@ -694,21 +714,25 @@ object GlobalDictionaryUtil {
carbonTable.getTableName).asScala.toArray
// generate global dict from pre defined column dict file
carbonLoadModel.initPredefDictMap()
-
val allDictionaryPath = carbonLoadModel.getAllDictPath
if (StringUtils.isEmpty(allDictionaryPath)) {
LOGGER.info("Generate global dictionary from source data files!")
// load data by using dataSource com.databricks.spark.csv
- var df = dataFrame.getOrElse(loadDataFrame(sqlContext, carbonLoadModel, hadoopConf))
- var headers = carbonLoadModel.getCsvHeaderColumns
- headers = headers.map(headerName => headerName.trim)
+ val headers = carbonLoadModel.getCsvHeaderColumns.map(_.trim)
val colDictFilePath = carbonLoadModel.getColDictFilePath
if (colDictFilePath != null) {
// generate predefined dictionary
generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
dimensions, carbonLoadModel, sqlContext, dictfolderPath)
}
- if (headers.length > df.columns.length) {
+
+ val headerOfInputData: Array[String] = if (dataFrame.isDefined) {
+ dataFrame.get.columns
+ } else {
+ headers
+ }
+
+ if (headers.length > headerOfInputData.length) {
val msg = "The number of columns in the file header do not match the " +
"number of columns in the data file; Either delimiter " +
"or fileheader provided is not correct"
@@ -717,14 +741,15 @@ object GlobalDictionaryUtil {
}
// use fact file to generate global dict
val (requireDimension, requireColumnNames) = pruneDimensions(dimensions,
- headers, df.columns)
+ headers, headerOfInputData)
if (requireDimension.nonEmpty) {
// select column to push down pruning
- df = df.select(requireColumnNames.head, requireColumnNames.tail: _*)
+ val dictRdd = loadInputDataAsDictRdd(sqlContext, carbonLoadModel, dataFrame,
+ requireColumnNames, hadoopConf)
val model = createDictionaryLoadModel(carbonLoadModel, carbonTableIdentifier,
requireDimension, dictfolderPath, false)
// combine distinct value in a block and partition by column
- val inputRDD = new CarbonBlockDistinctValuesCombineRDD(df.rdd, model)
+ val inputRDD = new CarbonBlockDistinctValuesCombineRDD(dictRdd, model)
.partitionBy(new ColumnPartitioner(model.primDimensions.length))
// generate global dictionary files
val statusList = new CarbonGlobalDictionaryGenerateRDD(inputRDD, model).collect()
[42/50] [abbrv] carbondata git commit: [CARBONDATA-1921]
Documentation - Merge Index
Posted by ja...@apache.org.
[CARBONDATA-1921] Documentation - Merge Index
add Merge Index document
This closes #1717
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/406d7544
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/406d7544
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/406d7544
Branch: refs/heads/fgdatamap
Commit: 406d754424fdd36c2710fa928afe9ced904b6805
Parents: 4b0eee3
Author: ajantha-bhat <aj...@gmail.com>
Authored: Fri Dec 22 16:17:43 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Dec 22 22:48:21 2017 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 4 ++--
docs/data-management-on-carbondata.md | 13 ++++++++++++-
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/406d7544/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 5875529..39f1e3b 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -70,7 +70,7 @@ This section provides the details of all the configurations required for CarbonD
| carbon.horizontal.UPDATE.compaction.threshold | 1 | This property specifies the threshold limit on number of UPDATE delta files within a segment. In case the number of delta files goes beyond the threshold, the UPDATE delta files within the segment becomes eligible for horizontal compaction and compacted into single UPDATE delta file. | Values between 1 to 10000. |
| carbon.horizontal.DELETE.compaction.threshold | 1 | This property specifies the threshold limit on number of DELETE delta files within a block of a segment. In case the number of delta files goes beyond the threshold, the DELETE delta files for the particular block of the segment becomes eligible for horizontal compaction and compacted into single DELETE delta file. | Values between 1 to 10000. |
| carbon.update.segment.parallelism | 1 | This property specifies the parallelism for each segment during update. If there are segments that contain too many records to update and the spark job encounter data-spill related errors, it is better to increase this property value. It is recommended to set this value to a multiple of the number of executors for balance. | Values between 1 to 1000. |
-
+| carbon.merge.index.in.segment | true | This property is used to merge all CarbonData index files (.carbonindex) inside a segment to a sinle CarbonData index merge file (.carbonindexmerge).| Values true or false |
* **Query Configuration**
@@ -222,4 +222,4 @@ RESET
* Success will be recorded in the driver log.
-* Failure will be displayed in the UI.
\ No newline at end of file
+* Failure will be displayed in the UI.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/406d7544/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 94820ad..90afb4e 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -231,6 +231,17 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
```
ALTER TABLE test_db.carbon CHANGE a1 a1 DECIMAL(18,2)
```
+- **MERGE INDEX**
+
+ This command is used to merge all the CarbonData index files (.carbonindex) inside a segment to a single CarbonData index merge file (.carbonindexmerge). This enhances the first query performance.
+ ```
+ ALTER TABLE [db_name.]table_name COMPACT 'SEGMENT_INDEX'
+ ```
+
+ Examples:
+ ```
+ ALTER TABLE test_db.carbon COMPACT 'SEGMENT_INDEX'
+ ```
### DROP TABLE
@@ -746,4 +757,4 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
Example:
```
DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.STARTTIME BEFORE '2017-06-01 12:05:06'
- ```
\ No newline at end of file
+ ```
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap]
Datamap FineGrain implementation
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 78544d3..fe0bbcf 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.datamap.examples;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
@@ -29,17 +28,18 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
-public class MinMaxDataWriter implements DataMapWriter {
+public class MinMaxDataWriter extends AbstractDataMapWriter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
@@ -50,17 +50,23 @@ public class MinMaxDataWriter implements DataMapWriter {
private Map<Integer, BlockletMinMax> blockMinMaxMap;
- private String blockPath;
+ private String dataWritePath;
+ public MinMaxDataWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
+ super(identifier, segmentId, dataWritePath);
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.dataWritePath = dataWritePath;
+ }
- @Override public void onBlockStart(String blockId, String blockPath) {
+ @Override public void onBlockStart(String blockId) {
pageLevelMax = null;
pageLevelMin = null;
blockletLevelMax = null;
blockletLevelMin = null;
blockMinMaxMap = null;
blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
- this.blockPath = blockPath;
}
@Override public void onBlockEnd(String blockId) {
@@ -161,7 +167,7 @@ public class MinMaxDataWriter implements DataMapWriter {
List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
tempMinMaxIndexBlockDetails = loadBlockDetails();
try {
- writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockId);
} catch (IOException ex) {
LOGGER.info(" Unable to write the file");
}
@@ -178,7 +184,6 @@ public class MinMaxDataWriter implements DataMapWriter {
tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
tmpminMaxIndexBlockDetails.setBlockletId(index);
- tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
}
return minMaxIndexBlockDetails;
@@ -187,22 +192,19 @@ public class MinMaxDataWriter implements DataMapWriter {
/**
* Write the data to a file. This is JSON format file.
* @param minMaxIndexBlockDetails
- * @param blockPath
* @param blockId
* @throws IOException
*/
public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
- String blockPath, String blockId) throws IOException {
- String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
- + ".minmaxindex";
+ String blockId) throws IOException {
+ String filePath = dataWritePath +"/" + blockId + ".minmaxindex";
BufferedWriter brWriter = null;
DataOutputStream dataOutStream = null;
try {
FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
Gson gsonObjectToWrite = new Gson();
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream, "UTF-8"));
String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
brWriter.write(minmaxIndexData);
} catch (IOException ioe) {
@@ -215,7 +217,11 @@ public class MinMaxDataWriter implements DataMapWriter {
dataOutStream.flush();
}
CarbonUtil.closeStreams(brWriter, dataOutStream);
+ commitFile(filePath);
}
}
+ @Override public void finish() throws IOException {
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
index 0596db5..93a453e 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -33,11 +33,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
private byte[][] maxValues;
/**
- * filePath pointing to the block.
- */
- private String filePath;
-
- /**
* BlockletID of the block.
*/
private Integer BlockletId;
@@ -59,14 +54,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
this.maxValues = maxValues;
}
- public String getFilePath() {
- return filePath;
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
public Integer getBlockletId() {
return BlockletId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 3e8ede2..2362b76 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -411,7 +411,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
}
carbonSplits.add(CarbonInputSplit.from(segmentId, fileSplit,
ColumnarFormatVersion.valueOf(
- CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION)));
+ CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION), null));
}
return carbonSplits;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index e89c2d6..3ac4642 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -84,6 +84,8 @@ public class CarbonInputSplit extends FileSplit
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ private String dataMapWritePath;
+
public CarbonInputSplit() {
segmentId = null;
taskId = "0";
@@ -94,7 +96,7 @@ public class CarbonInputSplit extends FileSplit
}
private CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
- ColumnarFormatVersion version, String[] deleteDeltaFiles) {
+ ColumnarFormatVersion version, String[] deleteDeltaFiles, String dataMapWritePath) {
super(path, start, length, locations);
this.segmentId = segmentId;
String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(path.getName());
@@ -106,11 +108,12 @@ public class CarbonInputSplit extends FileSplit
this.invalidSegments = new ArrayList<>();
this.version = version;
this.deleteDeltaFiles = deleteDeltaFiles;
+ this.dataMapWritePath = dataMapWritePath;
}
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
int numberOfBlocklets, ColumnarFormatVersion version, String[] deleteDeltaFiles) {
- this(segmentId, path, start, length, locations, version, deleteDeltaFiles);
+ this(segmentId, path, start, length, locations, version, deleteDeltaFiles, null);
this.numberOfBlocklets = numberOfBlocklets;
}
@@ -157,10 +160,10 @@ public class CarbonInputSplit extends FileSplit
}
public static CarbonInputSplit from(String segmentId, FileSplit split,
- ColumnarFormatVersion version)
+ ColumnarFormatVersion version, String dataMapWritePath)
throws IOException {
return new CarbonInputSplit(segmentId, split.getPath(), split.getStart(), split.getLength(),
- split.getLocations(), version, null);
+ split.getLocations(), version, null, dataMapWritePath);
}
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) {
@@ -174,6 +177,7 @@ public class CarbonInputSplit extends FileSplit
split.getLocations(), split.getLength(), blockletInfos, split.getVersion(),
split.getDeleteDeltaFiles());
blockInfo.setDetailInfo(split.getDetailInfo());
+ blockInfo.setDataMapWriterPath(split.dataMapWritePath);
tableBlockInfoList.add(blockInfo);
} catch (IOException e) {
throw new RuntimeException("fail to get location of split: " + split, e);
@@ -221,6 +225,10 @@ public class CarbonInputSplit extends FileSplit
detailInfo = new BlockletDetailInfo();
detailInfo.readFields(in);
}
+ boolean dataMapWriterPathExists = in.readBoolean();
+ if (dataMapWriterPathExists) {
+ dataMapWritePath = in.readUTF();
+ }
}
@Override public void write(DataOutput out) throws IOException {
@@ -242,6 +250,10 @@ public class CarbonInputSplit extends FileSplit
if (detailInfo != null) {
detailInfo.write(out);
}
+ out.writeBoolean(dataMapWritePath != null);
+ if (dataMapWritePath != null) {
+ out.writeUTF(dataMapWritePath);
+ }
}
public List<String> getInvalidSegments() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 22eca9f..5a72a02 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -35,6 +35,7 @@ import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -726,16 +727,17 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// get tokens for all the required FileSystem for table path
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { new Path(absoluteTableIdentifier.getTablePath()) }, job.getConfiguration());
-
- TableDataMap blockletMap = DataMapStoreManager.getInstance()
- .getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME,
- BlockletDataMapFactory.class.getName());
+ boolean distributedCG = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT));
+ TableDataMap blockletMap =
+ DataMapStoreManager.getInstance().chooseDataMap(absoluteTableIdentifier);
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
List<String> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
- if (dataMapJob != null) {
+ if (distributedCG || blockletMap.getDataMapFactory().getDataMapType() == DataMapType.FG) {
DistributableDataMapFormat datamapDstr =
- new DistributableDataMapFormat(absoluteTableIdentifier, BlockletDataMap.NAME,
+ new DistributableDataMapFormat(absoluteTableIdentifier, blockletMap.getDataMapName(),
segmentIds, partitionsToPrune,
BlockletDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
@@ -793,10 +795,12 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return null;
}
org.apache.carbondata.hadoop.CarbonInputSplit split =
- org.apache.carbondata.hadoop.CarbonInputSplit.from(blocklet.getSegmentId(),
+ org.apache.carbondata.hadoop.CarbonInputSplit.from(
+ blocklet.getSegmentId(),
new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(),
blocklet.getLocations()),
- ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()));
+ ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()),
+ blocklet.getDataMapWriterPath());
split.setDetailInfo(blocklet.getDetailInfo());
return split;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
new file mode 100644
index 0000000..5294de3
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileHolder
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.Blocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new CGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractCoarseGrainDataMap] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map {f =>
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event): Unit = {
+ ???
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class CGDataMap extends AbstractCoarseGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
+ var fileHolder: FileHolder = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = fileHolder.readInt(filePath, size-4)
+ val bytes = fileHolder.readByteArray(filePath, size-footerLen-4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f=>
+ new Blocklet(f._1, f._2+"")
+ }.asJava
+ }
+
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
+ }
+ tuples
+ }
+
+ private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear() = {
+ ???
+ }
+}
+
+class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String,
+ dataWritePath: String,
+ dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
+
+ var currentBlockId: String = null
+ val cgwritepath = dataWritePath + "/" +
+ dataMapName + System.nanoTime() + ".datamap"
+ lazy val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
+ val blockletList = new ArrayBuffer[Array[Byte]]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ maxMin +=
+ ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[Array[Byte]]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += newBytes
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ blockletList += sorted.head
+ blockletList += sorted.last
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(cgwritepath)
+ }
+
+
+}
+
+class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test cg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ sql(
+ """
+ | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[CGDataMapFactory].getName, "cgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 6f4e9cd..090371f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -20,25 +20,30 @@ package org.apache.carbondata.spark.testsuite.datamap
import java.util
import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{DataFrame, SaveMode}
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-class C2DataMapFactory() extends DataMapFactory {
+class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
+
+ var identifier: AbsoluteTableIdentifier = _
override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {}
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ }
override def fireEvent(event: Event): Unit = ???
@@ -46,13 +51,14 @@ class C2DataMapFactory() extends DataMapFactory {
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: String): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter =
+ DataMapWriterSuite.dataMapWriterC2Mock(identifier, segmentId, dataWritePath)
- override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, FilterType.EQUALTO)
+ override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
/**
* Get all distributable objects of a segmentid
@@ -62,6 +68,7 @@ class C2DataMapFactory() extends DataMapFactory {
override def toDistributable(segmentId: String): util.List[DataMapDistributable] = {
???
}
+
}
class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
@@ -166,9 +173,12 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
}
object DataMapWriterSuite {
+
var callbackSeq: Seq[String] = Seq[String]()
- val dataMapWriterC2Mock = new DataMapWriter {
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segmentId: String,
+ dataWritePath: String) =
+ new AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
override def onPageAdded(
blockletId: Int,
@@ -193,9 +203,21 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String, blockPath: String): Unit = {
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String) = {
callbackSeq :+= s"block start $blockId"
}
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish() = {
+
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
new file mode 100644
index 0000000..109364e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainDataMap, AbstractFineGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileHolder
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new FGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractFineGrainDataMap] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+ /**
+ * Get datamap for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractFineGrainDataMap]= {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event):Unit = {
+ ???
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class FGDataMap extends AbstractFineGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
+ var fileHolder: FileHolder = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = fileHolder.readInt(filePath, size - 4)
+ val bytes = fileHolder.readByteArray(filePath, size - footerLen - 4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties): java.util.List[FineGrainBlocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f =>
+ readAndFindData(f, value(0).getBytes())
+ }.filter(_.isDefined).map(_.get).asJava
+ }
+
+ private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
+ value: Array[Byte]): Option[FineGrainBlocklet] = {
+ val bytes = fileHolder.readByteArray(filePath, meta._4, meta._5)
+ val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(outputStream)
+ val blockletsData = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
+
+ import scala.collection.Searching._
+ val searching = blockletsData
+ .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
+ (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
+ override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
+ y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
+ }
+ })
+ if (searching.insertionPoint >= 0) {
+ val f = blockletsData(searching.insertionPoint)
+ val pages = f._3.zipWithIndex.map { p =>
+ val pg = new FineGrainBlocklet.Page
+ pg.setPageId(p._1)
+ pg.setRowId(f._2(p._2).toArray)
+ pg
+ }
+ pages
+ Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
+ } else {
+ None
+ }
+
+ }
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
+ }
+ tuples
+ }
+
+ def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear():Unit = {
+ ???
+ }
+}
+
+class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String, dataWriterPath: String, dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWriterPath) {
+
+ var currentBlockId: String = null
+ val fgwritepath = dataWriterPath + "/" + System.nanoTime() + ".datamap"
+ val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
+ val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
+ var position: Long = 0
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
+ var addedLast: Boolean = false
+ val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
+ addedLast = false
+ } else {
+ blockletListUpdated += oldValue
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletListUpdated += oldValue
+ }
+
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(blockletListUpdated)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ maxMin +=
+ ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
+ ._1), position, bytes.length))
+ position += bytes.length
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[(Array[Byte], Int)]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += ((newBytes, i))
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
+ var addedLast: Boolean = false
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
+ addedLast = false
+ } else {
+ blockletList += oldValue
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletList += oldValue
+ }
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(fgwritepath)
+ }
+
+
+}
+
+class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test fg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[FGDataMapFactory].getName, "fgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 09dbd71..6551aec 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -427,9 +427,10 @@ class CarbonScanRDD(
CarbonTableInputFormat.setQuerySegment(conf, identifier)
CarbonTableInputFormat.setFilterPredicates(conf, filterExpression)
CarbonTableInputFormat.setColumnProjection(conf, columnProjection)
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
- CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT).toBoolean) {
+ CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
+ if (CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT).toBoolean) {
CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 8e350d9..31a6701 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.datamap;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,8 +29,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.processing.store.TablePage;
@@ -43,25 +44,26 @@ public class DataMapWriterListener {
DataMapWriterListener.class.getCanonicalName());
// list indexed column name -> list of data map writer
- private Map<List<String>, List<DataMapWriter>> registry = new ConcurrentHashMap<>();
+ private Map<List<String>, List<AbstractDataMapWriter>> registry = new ConcurrentHashMap<>();
/**
* register all datamap writer for specified table and segment
*/
- public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId) {
+ public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(identifier);
if (tableDataMaps != null) {
for (TableDataMap tableDataMap : tableDataMaps) {
DataMapFactory factory = tableDataMap.getDataMapFactory();
- register(factory, segmentId);
+ register(factory, segmentId, dataWritePath);
}
}
}
/**
- * Register a DataMapWriter
+ * Register a AbstractDataMapWriter
*/
- private void register(DataMapFactory factory, String segmentId) {
+ private void register(DataMapFactory factory, String segmentId, String dataWritePath) {
assert (factory != null);
assert (segmentId != null);
DataMapMeta meta = factory.getMeta();
@@ -70,8 +72,8 @@ public class DataMapWriterListener {
return;
}
List<String> columns = factory.getMeta().getIndexedColumns();
- List<DataMapWriter> writers = registry.get(columns);
- DataMapWriter writer = factory.createWriter(segmentId);
+ List<AbstractDataMapWriter> writers = registry.get(columns);
+ AbstractDataMapWriter writer = factory.createWriter(segmentId, dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
@@ -79,36 +81,36 @@ public class DataMapWriterListener {
writers.add(writer);
registry.put(columns, writers);
}
- LOG.info("DataMapWriter " + writer + " added");
+ LOG.info("AbstractDataMapWriter " + writer + " added");
}
public void onBlockStart(String blockId, String blockPath) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId, blockPath);
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.onBlockStart(blockId);
}
}
}
public void onBlockEnd(String blockId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockEnd(blockId);
}
}
}
public void onBlockletStart(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletStart(blockletId);
}
}
}
public void onBlockletEnd(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletEnd(blockletId);
}
}
@@ -121,18 +123,29 @@ public class DataMapWriterListener {
* @param tablePage page data
*/
public void onPageAdded(int blockletId, int pageId, TablePage tablePage) {
- Set<Map.Entry<List<String>, List<DataMapWriter>>> entries = registry.entrySet();
- for (Map.Entry<List<String>, List<DataMapWriter>> entry : entries) {
+ Set<Map.Entry<List<String>, List<AbstractDataMapWriter>>> entries = registry.entrySet();
+ for (Map.Entry<List<String>, List<AbstractDataMapWriter>> entry : entries) {
List<String> indexedColumns = entry.getKey();
ColumnPage[] pages = new ColumnPage[indexedColumns.size()];
for (int i = 0; i < indexedColumns.size(); i++) {
pages[i] = tablePage.getColumnPage(indexedColumns.get(i));
}
- List<DataMapWriter> writers = entry.getValue();
- for (DataMapWriter writer : writers) {
+ List<AbstractDataMapWriter> writers = entry.getValue();
+ for (AbstractDataMapWriter writer : writers) {
writer.onPageAdded(blockletId, pageId, pages);
}
}
}
+ /**
+ * Finish all datamap writers
+ */
+ public void finish() throws IOException {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.finish();
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d15152c..497989a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
@@ -260,7 +261,8 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.sortScope = CarbonDataProcessorUtil.getSortScope(configuration);
DataMapWriterListener listener = new DataMapWriterListener();
- listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId());
+ listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId(),
+ storeLocation[new Random().nextInt(storeLocation.length)]);
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
carbonFactDataHandlerModel.writingCoresCount = configuration.getWritingCoresCount();
@@ -325,6 +327,12 @@ public class CarbonFactDataHandlerModel {
segmentProperties.getDimensions(),
segmentProperties.getMeasures());
+ DataMapWriterListener listener = new DataMapWriterListener();
+ listener.registerAllWriter(
+ loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(),
+ loadModel.getSegmentId(),
+ tempStoreLocation[new Random().nextInt(tempStoreLocation.length)]);
+ carbonFactDataHandlerModel.dataMapWriterlistener = listener;
return carbonFactDataHandlerModel;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65d05375/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index b00ec70..04e304f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.processing.store.writer;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
@@ -41,14 +39,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonMergerUtil;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -62,7 +57,6 @@ import org.apache.carbondata.format.IndexHeader;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.io.IOUtils;
public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
@@ -70,12 +64,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
LogServiceFactory.getLogService(AbstractFactDataWriter.class.getName());
/**
- * dfs.bytes-per-checksum
- * HDFS checksum length, block size for a file should be exactly divisible
- * by this value
- */
- private static final int HDFS_CHECKSUM_LENGTH = 512;
- /**
* file channel
*/
protected FileChannel fileChannel;
@@ -207,35 +195,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
/**
- * This method will return max of block size and file size
- *
- * @param blockSize
- * @param fileSize
- * @return
- */
- private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
- long maxSize = blockSize;
- if (fileSize > blockSize) {
- maxSize = fileSize;
- }
- // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
- // per checksum, dfs.bytes-per-checksum=512 must divide block size
- long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
- if (remainder > 0) {
- maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
- }
- // convert to make block size more readable.
- String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
- String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
- String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
- LOGGER.info(
- "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
- + readableFileSize + ", choose the max value " + readableMaxSize
- + " as the block size on HDFS");
- return maxSize;
- }
-
- /**
* This method will be used to update the file channel with new file if exceeding block size
* threshold, new file will be created once existing file reached the file size limit This
* method will first check whether existing file size is exceeded the file
@@ -296,7 +255,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
String fileName = this.carbonDataFileTempPath.substring(0,
this.carbonDataFileTempPath.lastIndexOf('.'));
if (copyInCurrentThread) {
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ fileName, dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
} else {
executorServiceSubmitList.add(executorService.submit(new CopyThread(fileName)));
}
@@ -449,7 +410,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
writer.close();
// copy from temp to actual store location
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(fileName,
+ dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
}
/**
@@ -459,20 +422,16 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws CarbonDataWriterException
*/
protected void closeExecutorService() throws CarbonDataWriterException {
- executorService.shutdown();
try {
+ listener.finish();
+ executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- }
- for (int i = 0; i < executorServiceSubmitList.size(); i++) {
- try {
+ for (int i = 0; i < executorServiceSubmitList.size(); i++) {
executorServiceSubmitList.get(i).get();
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- } catch (ExecutionException e) {
- throw new CarbonDataWriterException(e.getMessage());
}
+ } catch (InterruptedException | ExecutionException | IOException e) {
+ LOGGER.error(e, "Error while finishing writer");
+ throw new CarbonDataWriterException(e.getMessage());
}
}
@@ -490,62 +449,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
}
- /**
- * This method will copy the given file to carbon store location
- *
- * @param localFileName local file name with full path
- * @throws CarbonDataWriterException
- */
- protected void copyCarbonDataFileToCarbonStorePath(String localFileName)
- throws CarbonDataWriterException {
- long copyStartTime = System.currentTimeMillis();
- LOGGER.info("Copying " + localFileName + " --> " + dataWriterVo.getCarbonDataDirectoryPath());
- try {
- CarbonFile localCarbonFile =
- FileFactory.getCarbonFile(localFileName, FileFactory.getFileType(localFileName));
- String carbonFilePath = dataWriterVo.getCarbonDataDirectoryPath() + localFileName
- .substring(localFileName.lastIndexOf(File.separator));
- copyLocalFileToCarbonStore(carbonFilePath, localFileName,
- CarbonCommonConstants.BYTEBUFFER_SIZE,
- getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
- } catch (IOException e) {
- throw new CarbonDataWriterException(
- "Problem while copying file from local store to carbon store", e);
- }
- LOGGER.info(
- "Total copy time (ms) to copy file " + localFileName + " is " + (System.currentTimeMillis()
- - copyStartTime));
- }
-
- /**
- * This method will read the local carbon data file and write to carbon data file in HDFS
- *
- * @param carbonStoreFilePath
- * @param localFilePath
- * @param bufferSize
- * @param blockSize
- * @throws IOException
- */
- private void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
- int bufferSize, long blockSize) throws IOException {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- try {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
- + " (bytes");
- }
- dataOutputStream = FileFactory
- .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
- bufferSize, blockSize);
- dataInputStream = FileFactory
- .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
- IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
- } finally {
- CarbonUtil.closeStream(dataInputStream);
- CarbonUtil.closeStream(dataOutputStream);
- }
- }
/**
* This method will copy the carbon data file from local store location to
@@ -570,7 +473,10 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws Exception if unable to compute a result
*/
@Override public Void call() throws Exception {
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ fileName,
+ dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
return null;
}
[06/50] [abbrv] carbondata git commit: [CARBONDATA-1855][PARTITION]
Added outputformat to carbon
Posted by ja...@apache.org.
[CARBONDATA-1855][PARTITION] Added outputformat to carbon
Support standard Hadoop outputformat interface for carbon. This PR supports table level output format: CarbonTableOutputFormat, It will be helpful for integrations to execution engines like the spark, hive, and presto.
It should maintain segment management as well while writing the data to support incremental loading feature.
This closes #1642
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/91e6f6f4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/91e6f6f4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/91e6f6f4
Branch: refs/heads/fgdatamap
Commit: 91e6f6f43b915b7ddc52f36233526aa2f5a77782
Parents: 9f332be
Author: ravipesala <ra...@gmail.com>
Authored: Mon Dec 4 16:07:03 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Dec 17 13:30:55 2017 +0800
----------------------------------------------------------------------
.../core/metadata/datatype/StructType.java | 2 +-
.../apache/carbondata/core/util/CarbonUtil.java | 15 -
.../hadoop/api/CarbonOutputCommitter.java | 94 +++++
.../hadoop/api/CarbonTableOutputFormat.java | 349 ++++++++++++++++++-
.../hadoop/ft/CarbonOutputMapperTest.java | 114 ++++++
.../hadoop/test/util/StoreCreator.java | 32 +-
.../carbondata/spark/util/CommonUtil.scala | 45 ---
.../spark/rdd/CarbonDataRDDFactory.scala | 8 +-
.../management/CarbonLoadDataCommand.scala | 7 +-
.../iterator/CarbonOutputIteratorWrapper.java | 128 +++++++
.../loading/model/CarbonLoadModel.java | 9 +
.../processing/merger/CarbonDataMergerUtil.java | 4 +-
.../processing/util/CarbonLoaderUtil.java | 68 +++-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
14 files changed, 772 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
index 6417f37..97cc4f0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.metadata.datatype;
import java.util.List;
-class StructType extends DataType {
+public class StructType extends DataType {
private List<StructField> fields;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 148098d..910efea 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2156,21 +2156,6 @@ public final class CarbonUtil {
return parentPath.toString() + CarbonCommonConstants.FILE_SEPARATOR + newTableName;
}
- /*
- * This method will add data size and index size into tablestatus for each segment
- */
- public static void addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
- String segmentId, CarbonTable carbonTable) throws IOException {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
- Map<String, Long> dataIndexSize =
- CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
- loadMetadataDetails
- .setDataSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString());
- loadMetadataDetails
- .setIndexSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).toString());
- }
-
/**
* This method will calculate the data size and index size for carbon table
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
new file mode 100644
index 0000000..9bcb2be
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.processing.util.CarbonLoaderUtil;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+
+/**
+ * Outputcommitter which manages the segments during loading.It commits segment information to the
+ * tablestatus file upon success or fail.
+ */
+public class CarbonOutputCommitter extends FileOutputCommitter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonOutputCommitter.class.getName());
+
+ public CarbonOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ }
+
+ /**
+ * Update the tablestatus with inprogress while setup the job.
+ *
+ * @param context
+ * @throws IOException
+ */
+ @Override public void setupJob(JobContext context) throws IOException {
+ super.setupJob(context);
+ boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
+ CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(loadModel, overwriteSet);
+ CarbonTableOutputFormat.setLoadModel(context.getConfiguration(), loadModel);
+ }
+
+ /**
+ * Update the tablestatus as success after job is success
+ *
+ * @param context
+ * @throws IOException
+ */
+ @Override public void commitJob(JobContext context) throws IOException {
+ super.commitJob(context);
+ boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
+ CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
+ CarbonLoaderUtil.populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS,
+ loadModel.getFactTimeStamp(), true);
+ CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(),
+ loadModel.getCarbonDataLoadSchema().getCarbonTable());
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
+ }
+
+ /**
+ * Update the tablestatus as fail if any fail happens.
+ *
+ * @param context
+ * @param state
+ * @throws IOException
+ */
+ @Override public void abortJob(JobContext context, JobStatus.State state) throws IOException {
+ super.abortJob(context, state);
+ CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
+ LOGGER.error("Loading failed with job status : " + state);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 7c9b3ed..9504502 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -18,22 +18,349 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
+import java.util.List;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.util.Progressable;
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.datatype.StructType;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
+import org.apache.carbondata.processing.loading.DataLoadExecutor;
+import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
+import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
+import org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
- * Base class for all output format for CarbonData file.
- * @param <T>
+ * This is table level output format which writes the data to store in new segment. Each load
+ * creates new segment folder and manages the folder through tablestatus file.
+ * It also generate and writes dictionary data during load only if dictionary server is configured.
*/
-public abstract class CarbonTableOutputFormat<T> extends FileOutputFormat<Void, T> {
+// TODO Move dictionary generater which is coded in spark to MR framework.
+public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, StringArrayWritable> {
- @Override
- public RecordWriter<Void, T> getRecordWriter(FileSystem ignored, JobConf job, String name,
- Progressable progress) throws IOException {
+ private static final String LOAD_MODEL = "mapreduce.carbontable.load.model";
+ private static final String DATABASE_NAME = "mapreduce.carbontable.databaseName";
+ private static final String TABLE_NAME = "mapreduce.carbontable.tableName";
+ private static final String TABLE = "mapreduce.carbontable.table";
+ private static final String TABLE_PATH = "mapreduce.carbontable.tablepath";
+ private static final String INPUT_SCHEMA = "mapreduce.carbontable.inputschema";
+ private static final String TEMP_STORE_LOCATIONS = "mapreduce.carbontable.tempstore.locations";
+ private static final String OVERWRITE_SET = "mapreduce.carbontable.set.overwrite";
+ public static final String COMPLEX_DELIMITERS = "mapreduce.carbontable.complex_delimiters";
+ public static final String SERIALIZATION_NULL_FORMAT =
+ "mapreduce.carbontable.serialization.null.format";
+ public static final String BAD_RECORDS_LOGGER_ENABLE =
+ "mapreduce.carbontable.bad.records.logger.enable";
+ public static final String BAD_RECORDS_LOGGER_ACTION =
+ "mapreduce.carbontable.bad.records.logger.action";
+ public static final String IS_EMPTY_DATA_BAD_RECORD =
+ "mapreduce.carbontable.empty.data.bad.record";
+ public static final String SKIP_EMPTY_LINE = "mapreduce.carbontable.skip.empty.line";
+ public static final String SORT_SCOPE = "mapreduce.carbontable.load.sort.scope";
+ public static final String BATCH_SORT_SIZE_INMB =
+ "mapreduce.carbontable.batch.sort.size.inmb";
+ public static final String GLOBAL_SORT_PARTITIONS =
+ "mapreduce.carbontable.global.sort.partitions";
+ public static final String BAD_RECORD_PATH = "mapreduce.carbontable.bad.record.path";
+ public static final String DATE_FORMAT = "mapreduce.carbontable.date.format";
+ public static final String TIMESTAMP_FORMAT = "mapreduce.carbontable.timestamp.format";
+ public static final String IS_ONE_PASS_LOAD = "mapreduce.carbontable.one.pass.load";
+ public static final String DICTIONARY_SERVER_HOST =
+ "mapreduce.carbontable.dict.server.host";
+ public static final String DICTIONARY_SERVER_PORT =
+ "mapreduce.carbontable.dict.server.port";
+
+ private CarbonOutputCommitter committer;
+
+ public static void setDatabaseName(Configuration configuration, String databaseName) {
+ if (null != databaseName) {
+ configuration.set(DATABASE_NAME, databaseName);
+ }
+ }
+
+ public static String getDatabaseName(Configuration configuration) {
+ return configuration.get(DATABASE_NAME);
+ }
+
+ public static void setTableName(Configuration configuration, String tableName) {
+ if (null != tableName) {
+ configuration.set(TABLE_NAME, tableName);
+ }
+ }
+
+ public static String getTableName(Configuration configuration) {
+ return configuration.get(TABLE_NAME);
+ }
+
+ public static void setTablePath(Configuration configuration, String tablePath) {
+ if (null != tablePath) {
+ configuration.set(TABLE_PATH, tablePath);
+ }
+ }
+
+ public static String getTablePath(Configuration configuration) {
+ return configuration.get(TABLE_PATH);
+ }
+
+ public static void setCarbonTable(Configuration configuration, CarbonTable carbonTable)
+ throws IOException {
+ if (carbonTable != null) {
+ configuration.set(TABLE,
+ ObjectSerializationUtil.convertObjectToString(carbonTable.getTableInfo().serialize()));
+ }
+ }
+
+ public static CarbonTable getCarbonTable(Configuration configuration) throws IOException {
+ CarbonTable carbonTable = null;
+ String encodedString = configuration.get(TABLE);
+ if (encodedString != null) {
+ byte[] bytes = (byte[]) ObjectSerializationUtil.convertStringToObject(encodedString);
+ TableInfo tableInfo = TableInfo.deserialize(bytes);
+ carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
+ }
+ return carbonTable;
+ }
+
+ public static void setLoadModel(Configuration configuration, CarbonLoadModel loadModel)
+ throws IOException {
+ if (loadModel != null) {
+ configuration.set(LOAD_MODEL, ObjectSerializationUtil.convertObjectToString(loadModel));
+ }
+ }
+
+ public static void setInputSchema(Configuration configuration, StructType inputSchema)
+ throws IOException {
+ if (inputSchema != null && inputSchema.getFields().size() > 0) {
+ configuration.set(INPUT_SCHEMA, ObjectSerializationUtil.convertObjectToString(inputSchema));
+ } else {
+ throw new UnsupportedOperationException("Input schema must be set");
+ }
+ }
+
+ private static StructType getInputSchema(Configuration configuration) throws IOException {
+ String encodedString = configuration.get(INPUT_SCHEMA);
+ if (encodedString != null) {
+ return (StructType) ObjectSerializationUtil.convertStringToObject(encodedString);
+ }
return null;
}
+
+ public static boolean isOverwriteSet(Configuration configuration) {
+ String overwrite = configuration.get(OVERWRITE_SET);
+ if (overwrite != null) {
+ return Boolean.parseBoolean(overwrite);
+ }
+ return false;
+ }
+
+ public static void setOverwrite(Configuration configuration, boolean overwrite) {
+ configuration.set(OVERWRITE_SET, String.valueOf(overwrite));
+ }
+
+ public static void setTempStoreLocations(Configuration configuration, String[] tempLocations)
+ throws IOException {
+ if (tempLocations != null && tempLocations.length > 0) {
+ configuration
+ .set(TEMP_STORE_LOCATIONS, ObjectSerializationUtil.convertObjectToString(tempLocations));
+ }
+ }
+
+ private static String[] getTempStoreLocations(TaskAttemptContext taskAttemptContext)
+ throws IOException {
+ String encodedString = taskAttemptContext.getConfiguration().get(TEMP_STORE_LOCATIONS);
+ if (encodedString != null) {
+ return (String[]) ObjectSerializationUtil.convertStringToObject(encodedString);
+ }
+ return new String[] {
+ System.getProperty("java.io.tmpdir") + "/" + System.nanoTime() + "_" + taskAttemptContext
+ .getTaskAttemptID().toString() };
+ }
+
+ @Override
+ public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext context)
+ throws IOException {
+ if (this.committer == null) {
+ Path output = getOutputPath(context);
+ this.committer = new CarbonOutputCommitter(output, context);
+ }
+ return this.committer;
+ }
+
+ @Override
+ public RecordWriter<NullWritable, StringArrayWritable> getRecordWriter(
+ TaskAttemptContext taskAttemptContext) throws IOException {
+ final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
+ loadModel.setTaskNo(taskAttemptContext.getTaskAttemptID().getTaskID().getId() + "");
+ final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
+ final CarbonOutputIteratorWrapper iteratorWrapper = new CarbonOutputIteratorWrapper();
+ final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
+ CarbonRecordWriter recordWriter = new CarbonRecordWriter(iteratorWrapper, dataLoadExecutor);
+ new Thread() {
+ @Override public void run() {
+ try {
+ dataLoadExecutor.execute(
+ loadModel,
+ tempStoreLocations,
+ new CarbonIterator[] { iteratorWrapper });
+ } catch (Exception e) {
+ dataLoadExecutor.close();
+ throw new RuntimeException(e);
+ }
+ }
+ }.start();
+
+ return recordWriter;
+ }
+
+ public static CarbonLoadModel getLoadModel(Configuration conf) throws IOException {
+ CarbonLoadModel model;
+ String encodedString = conf.get(LOAD_MODEL);
+ if (encodedString != null) {
+ model = (CarbonLoadModel) ObjectSerializationUtil.convertStringToObject(encodedString);
+ return model;
+ }
+ model = new CarbonLoadModel();
+ CarbonProperties carbonProperty = CarbonProperties.getInstance();
+ model.setDatabaseName(CarbonTableOutputFormat.getDatabaseName(conf));
+ model.setTableName(CarbonTableOutputFormat.getTableName(conf));
+ model.setCarbonDataLoadSchema(new CarbonDataLoadSchema(getCarbonTable(conf)));
+ model.setTablePath(getTablePath(conf));
+
+ setFileHeader(conf, model);
+ model.setSerializationNullFormat(conf.get(SERIALIZATION_NULL_FORMAT, "\\N"));
+ model.setBadRecordsLoggerEnable(
+ conf.get(
+ BAD_RECORDS_LOGGER_ENABLE,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)));
+ model.setBadRecordsAction(
+ conf.get(
+ BAD_RECORDS_LOGGER_ACTION,
+ carbonProperty.getProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)));
+
+ model.setIsEmptyDataBadRecord(
+ conf.get(
+ IS_EMPTY_DATA_BAD_RECORD,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)));
+
+ model.setSkipEmptyLine(
+ conf.get(
+ SKIP_EMPTY_LINE,
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
+
+ String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\$" + "," + "\\:");
+ String[] split = complexDelim.split(",");
+ model.setComplexDelimiterLevel1(split[0]);
+ if (split.length > 1) {
+ model.setComplexDelimiterLevel1(split[1]);
+ }
+ model.setDateFormat(
+ conf.get(
+ DATE_FORMAT,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)));
+
+ model.setTimestampformat(
+ conf.get(
+ TIMESTAMP_FORMAT,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)));
+
+ model.setGlobalSortPartitions(
+ conf.get(
+ GLOBAL_SORT_PARTITIONS,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ null)));
+
+ model.setBatchSortSizeInMb(
+ conf.get(
+ BATCH_SORT_SIZE_INMB,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperty.getProperty(
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))));
+
+ model.setBadRecordsLocation(
+ conf.get(BAD_RECORD_PATH,
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperty.getProperty(
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))));
+
+ model.setUseOnePass(
+ conf.getBoolean(IS_ONE_PASS_LOAD,
+ Boolean.parseBoolean(
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT))));
+ return model;
+ }
+
+ private static void setFileHeader(Configuration configuration, CarbonLoadModel model)
+ throws IOException {
+ StructType inputSchema = getInputSchema(configuration);
+ if (inputSchema == null || inputSchema.getFields().size() == 0) {
+ throw new UnsupportedOperationException("Input schema must be set");
+ }
+ List<StructField> fields = inputSchema.getFields();
+ StringBuilder builder = new StringBuilder();
+ String[] columns = new String[fields.size()];
+ int i = 0;
+ for (StructField field : fields) {
+ builder.append(field.getFieldName());
+ builder.append(",");
+ columns[i++] = field.getFieldName();
+ }
+ String header = builder.toString();
+ model.setCsvHeader(header.substring(0, header.length() - 1));
+ model.setCsvHeaderColumns(columns);
+ }
+
+ private static class CarbonRecordWriter extends RecordWriter<NullWritable, StringArrayWritable> {
+
+ private CarbonOutputIteratorWrapper iteratorWrapper;
+
+ private DataLoadExecutor dataLoadExecutor;
+
+ public CarbonRecordWriter(CarbonOutputIteratorWrapper iteratorWrapper,
+ DataLoadExecutor dataLoadExecutor) {
+ this.iteratorWrapper = iteratorWrapper;
+ this.dataLoadExecutor = dataLoadExecutor;
+ }
+
+ @Override
+ public void write(NullWritable aVoid, StringArrayWritable strings)
+ throws InterruptedException {
+ iteratorWrapper.write(strings.get());
+ }
+
+ @Override
+ public void close(TaskAttemptContext taskAttemptContext) {
+ iteratorWrapper.close();
+ dataLoadExecutor.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
new file mode 100644
index 0000000..006ffd2
--- /dev/null
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonOutputMapperTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.hadoop.ft;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
+import org.apache.carbondata.hadoop.test.util.StoreCreator;
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
+import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.Test;
+
+public class CarbonOutputMapperTest extends TestCase {
+
+ CarbonLoadModel carbonLoadModel;
+
+ // changed setUp to static init block to avoid un wanted multiple time store creation
+ static {
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords");
+ }
+
+
+ @Test public void testOutputFormat() throws Exception {
+ runJob("");
+ String segmentPath = CarbonTablePath.getSegmentPath(carbonLoadModel.getTablePath(), "0");
+ File file = new File(segmentPath);
+ assert (file.exists());
+ File[] listFiles = file.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.endsWith(".carbondata") || name.endsWith(".carbonindex");
+ }
+ });
+
+ assert (listFiles.length == 2);
+
+ }
+
+
+ @Override public void tearDown() throws Exception {
+ super.tearDown();
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true");
+ }
+
+ @Override public void setUp() throws Exception {
+ super.setUp();
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "false");
+ carbonLoadModel = StoreCreator.getCarbonLoadModel();
+ }
+
+ public static class Map extends Mapper<NullWritable, StringArrayWritable, NullWritable, StringArrayWritable> {
+
+ @Override protected void map(NullWritable key, StringArrayWritable value, Context context)
+ throws IOException, InterruptedException {
+ context.write(key, value);
+ }
+ }
+
+ private void runJob(String outPath) throws Exception {
+ Configuration configuration = new Configuration();
+ configuration.set("mapreduce.cluster.local.dir", new File(outPath + "1").getCanonicalPath());
+ Job job = Job.getInstance(configuration);
+ job.setJarByClass(CarbonOutputMapperTest.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(StringArrayWritable.class);
+ job.setMapperClass(Map.class);
+ job.setNumReduceTasks(0);
+
+ FileInputFormat.addInputPath(job, new Path(carbonLoadModel.getFactFilePath()));
+ CarbonTableOutputFormat.setLoadModel(job.getConfiguration(), carbonLoadModel);
+ CarbonTableOutputFormat.setCarbonTable(job.getConfiguration(), carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable());
+ CSVInputFormat.setHeaderExtractionEnabled(job.getConfiguration(), true);
+ job.setInputFormatClass(CSVInputFormat.class);
+ job.setOutputFormatClass(CarbonTableOutputFormat.class);
+ CarbonUtil.deleteFoldersAndFiles(new File(carbonLoadModel.getTablePath() + "1"));
+ FileOutputFormat.setOutputPath(job, new Path(carbonLoadModel.getTablePath() + "1"));
+ job.getConfiguration().set("outpath", outPath);
+ job.getConfiguration().set("query.id", String.valueOf(System.nanoTime()));
+ job.waitForCompletion(true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 531bed5..d3fd087 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -164,17 +164,7 @@ public class StoreCreator {
*/
public static void createCarbonStore() {
try {
- String factFilePath =
- new File("../hadoop/src/test/resources/data.csv").getCanonicalPath();
- File storeDir = new File(storePath);
- CarbonUtil.deleteFoldersAndFiles(storeDir);
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION_HDFS,
- storePath);
-
- CarbonTable table = createTable(absoluteTableIdentifier);
- writeDictionary(factFilePath, table);
- CarbonLoadModel loadModel =
- buildCarbonLoadModel(table, factFilePath, absoluteTableIdentifier);
+ CarbonLoadModel loadModel = getCarbonLoadModel();
executeGraph(loadModel, storePath);
@@ -183,6 +173,19 @@ public class StoreCreator {
}
}
+ public static CarbonLoadModel getCarbonLoadModel() throws Exception {
+ String factFilePath =
+ new File("../hadoop/src/test/resources/data.csv").getCanonicalPath();
+ File storeDir = new File(storePath);
+ CarbonUtil.deleteFoldersAndFiles(storeDir);
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION_HDFS,
+ storePath);
+
+ CarbonTable table = createTable(absoluteTableIdentifier);
+ writeDictionary(factFilePath, table);
+ return buildCarbonLoadModel(table, factFilePath, absoluteTableIdentifier);
+ }
+
public static CarbonTable createTable(
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
TableInfo tableInfo = new TableInfo();
@@ -198,6 +201,7 @@ public class StoreCreator {
id.setDataType(DataTypes.INT);
id.setEncodingList(encodings);
id.setColumnUniqueId(UUID.randomUUID().toString());
+ id.setColumnReferenceId(id.getColumnUniqueId());
id.setDimensionColumn(true);
id.setColumnGroup(1);
columnSchemas.add(id);
@@ -211,6 +215,7 @@ public class StoreCreator {
date.setDimensionColumn(true);
date.setColumnGroup(2);
date.setSortColumn(true);
+ date.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(date);
ColumnSchema country = new ColumnSchema();
@@ -222,6 +227,7 @@ public class StoreCreator {
country.setDimensionColumn(true);
country.setColumnGroup(3);
country.setSortColumn(true);
+ country.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(country);
ColumnSchema name = new ColumnSchema();
@@ -233,6 +239,7 @@ public class StoreCreator {
name.setDimensionColumn(true);
name.setColumnGroup(4);
name.setSortColumn(true);
+ name.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(name);
ColumnSchema phonetype = new ColumnSchema();
@@ -244,6 +251,7 @@ public class StoreCreator {
phonetype.setDimensionColumn(true);
phonetype.setColumnGroup(5);
phonetype.setSortColumn(true);
+ phonetype.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(phonetype);
ColumnSchema serialname = new ColumnSchema();
@@ -255,6 +263,7 @@ public class StoreCreator {
serialname.setDimensionColumn(true);
serialname.setColumnGroup(6);
serialname.setSortColumn(true);
+ serialname.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(serialname);
ColumnSchema salary = new ColumnSchema();
@@ -264,6 +273,7 @@ public class StoreCreator {
salary.setEncodingList(new ArrayList<Encoding>());
salary.setColumnUniqueId(UUID.randomUUID().toString());
salary.setDimensionColumn(false);
+ salary.setColumnReferenceId(id.getColumnUniqueId());
salary.setColumnGroup(7);
columnSchemas.add(salary);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 943c0a5..ab47532 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -514,51 +514,6 @@ object CommonUtil {
parsedPropertyValueString
}
- def readAndUpdateLoadProgressInTableMeta(model: CarbonLoadModel,
- insertOverwrite: Boolean): Unit = {
- val newLoadMetaEntry = new LoadMetadataDetails
- val status = if (insertOverwrite) {
- SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
- } else {
- SegmentStatus.INSERT_IN_PROGRESS
- }
-
- // reading the start time of data load.
- val loadStartTime = CarbonUpdateUtil.readCurrentTime
- model.setFactTimeStamp(loadStartTime)
- CarbonLoaderUtil.populateNewLoadMetaEntry(
- newLoadMetaEntry, status, model.getFactTimeStamp, false)
- val entryAdded: Boolean =
- CarbonLoaderUtil.recordNewLoadMetadata(newLoadMetaEntry, model, true, insertOverwrite)
- if (!entryAdded) {
- sys.error(s"Failed to add entry in table status for " +
- s"${ model.getDatabaseName }.${model.getTableName}")
- }
- }
-
- /**
- * This method will update the load failure entry in the table status file
- *
- * @param model
- */
- def updateTableStatusForFailure(
- model: CarbonLoadModel): Unit = {
- // in case if failure the load status should be "Marked for delete" so that it will be taken
- // care during clean up
- val loadStatus = SegmentStatus.MARKED_FOR_DELETE
- // always the last entry in the load metadata details will be the current load entry
- val loadMetaEntry = model.getLoadMetadataDetails.get(model.getLoadMetadataDetails.size - 1)
- CarbonLoaderUtil
- .populateNewLoadMetaEntry(loadMetaEntry, loadStatus, model.getFactTimeStamp, true)
- val updationStatus = CarbonLoaderUtil.recordNewLoadMetadata(loadMetaEntry, model, false, false)
- if (!updationStatus) {
- sys
- .error(s"Failed to update failure entry in table status for ${
- model
- .getDatabaseName
- }.${ model.getTableName }")
- }
- }
def readLoadMetadataDetails(model: CarbonLoadModel): Unit = {
val metadataPath = model.getCarbonDataLoadSchema.getCarbonTable.getMetaDataFilepath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 4933a45..d6360a7 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -473,7 +473,7 @@ object CarbonDataRDDFactory {
}
if (loadStatus == SegmentStatus.LOAD_FAILURE) {
// update the load entry in table status file for changing the status to marked for delete
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.info("********starting clean up**********")
CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
LOGGER.info("********clean up done**********")
@@ -488,7 +488,7 @@ object CarbonDataRDDFactory {
status(0)._2._2.failureCauses == FailureCauses.BAD_RECORDS &&
carbonLoadModel.getBadRecordsAction.split(",")(1) == LoggerAction.FAIL.name) {
// update the load entry in table status file for changing the status to marked for delete
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.info("********starting clean up**********")
CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
LOGGER.info("********clean up done**********")
@@ -527,7 +527,7 @@ object CarbonDataRDDFactory {
newEntryLoadStatus,
overwriteTable)
if (!done) {
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.info("********starting clean up**********")
CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
LOGGER.info("********clean up done**********")
@@ -800,7 +800,7 @@ object CarbonDataRDDFactory {
newEntryLoadStatus,
carbonLoadModel.getFactTimeStamp,
true)
- CarbonUtil
+ CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(metadataDetails, carbonLoadModel.getSegmentId, carbonTable)
val done = CarbonLoaderUtil.recordNewLoadMetadata(metadataDetails, carbonLoadModel, false,
overwriteTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 0c6aeda..58671b7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -46,6 +46,7 @@ import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DictionaryLoadModel}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataLoadingUtil, GlobalDictionaryUtil}
@@ -153,7 +154,7 @@ case class CarbonLoadDataCommand(
GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
// add the start entry for the new load in the table status file
if (updateModel.isEmpty) {
- CommonUtil.readAndUpdateLoadProgressInTableMeta(carbonLoadModel, isOverwriteTable)
+ CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(carbonLoadModel, isOverwriteTable)
}
if (isOverwriteTable) {
LOGGER.info(s"Overwrite of carbon table with $dbName.$tableName is in progress")
@@ -197,12 +198,12 @@ case class CarbonLoadDataCommand(
} catch {
case CausedBy(ex: NoRetryException) =>
// update the load entry in table status file for changing the status to marked for delete
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.error(ex, s"Dataload failure for $dbName.$tableName")
throw new RuntimeException(s"Dataload failure for $dbName.$tableName, ${ex.getMessage}")
case ex: Exception =>
// update the load entry in table status file for changing the status to marked for delete
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.error(ex)
LOGGER.audit(s"Dataload failure for $dbName.$tableName. Please check the logs")
throw ex
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
new file mode 100644
index 0000000..abe90f1
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.iterator;
+
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.carbondata.common.CarbonIterator;
+
+/**
+ * It is wrapper class to hold the rows in batches when record writer writes the data and allows
+ * to iterate on it during data load. It uses blocking queue to coordinate between read and write.
+ */
+public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
+
+ private boolean close = false;
+
+ /**
+ * Number of rows kept in memory at most will be batchSize * queue size
+ */
+ private int batchSize = 1000;
+
+ private RowBatch loadBatch = new RowBatch(batchSize);
+
+ private RowBatch readBatch;
+
+ private ArrayBlockingQueue<RowBatch> queue = new ArrayBlockingQueue<>(10);
+
+ public void write(String[] row) throws InterruptedException {
+ if (!loadBatch.addRow(row)) {
+ loadBatch.readyRead();
+ queue.put(loadBatch);
+ loadBatch = new RowBatch(batchSize);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !queue.isEmpty() || !close || readBatch != null && readBatch.hasNext();
+ }
+
+ @Override
+ public String[] next() {
+ if (readBatch == null || !readBatch.hasNext()) {
+ try {
+ readBatch = queue.take();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return readBatch.next();
+ }
+
+ @Override
+ public void close() {
+ if (loadBatch.isLoading()) {
+ try {
+ loadBatch.readyRead();
+ queue.put(loadBatch);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ close = true;
+ }
+
+ private static class RowBatch extends CarbonIterator<String[]> {
+
+ private int counter;
+
+ private String[][] batch;
+
+ private int size;
+
+ private boolean isLoading = true;
+
+ private RowBatch(int size) {
+ batch = new String[size][];
+ this.size = size;
+ }
+
+ /**
+ * Add row to the batch, it can hold rows till the batch size.
+ * @param row
+ * @return false if the row cannot be added as batch is full.
+ */
+ public boolean addRow(String[] row) {
+ batch[counter++] = row;
+ return counter < size;
+ }
+
+ public void readyRead() {
+ size = counter;
+ counter = 0;
+ isLoading = false;
+ }
+
+ public boolean isLoading() {
+ return isLoading;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return counter < size;
+ }
+
+ @Override
+ public String[] next() {
+ assert (counter < size);
+ return batch[counter++];
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index c85d50f..7b952e3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -525,6 +525,15 @@ public class CarbonLoadModel implements Serializable {
}
/**
+ * Get the current load metadata.
+ *
+ * @return
+ */
+ public LoadMetadataDetails getCurrentLoadMetadataDetail() {
+ return loadMetadataDetails.get(loadMetadataDetails.size() - 1);
+ }
+
+ /**
* setLoadMetadataDetails.
*
* @param loadMetadataDetails
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index d6f2d9a..3729b1d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -50,11 +50,11 @@ import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.processing.util.CarbonLoaderUtil;
/**
* utility class for load merging.
@@ -327,7 +327,7 @@ public final class CarbonDataMergerUtil {
loadMetadataDetails.setLoadEndTime(loadEnddate);
CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
loadMetadataDetails.setLoadName(mergedLoadNumber);
- CarbonUtil
+ CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable);
loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
loadMetadataDetails.setPartitionCount("0");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index b0c690b..10fdd31 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -24,17 +24,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -158,7 +148,7 @@ public final class CarbonLoaderUtil {
*/
public static boolean recordNewLoadMetadata(LoadMetadataDetails newMetaEntry,
CarbonLoadModel loadModel, boolean loadStartEntry, boolean insertOverwrite)
- throws IOException, InterruptedException {
+ throws IOException {
boolean status = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -373,6 +363,46 @@ public final class CarbonLoaderUtil {
}
+ public static void readAndUpdateLoadProgressInTableMeta(CarbonLoadModel model,
+ boolean insertOverwrite) throws IOException {
+ LoadMetadataDetails newLoadMetaEntry = new LoadMetadataDetails();
+ SegmentStatus status = SegmentStatus.INSERT_IN_PROGRESS;
+ if (insertOverwrite) {
+ status = SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS;
+ }
+
+ // reading the start time of data load.
+ long loadStartTime = CarbonUpdateUtil.readCurrentTime();
+ model.setFactTimeStamp(loadStartTime);
+ CarbonLoaderUtil
+ .populateNewLoadMetaEntry(newLoadMetaEntry, status, model.getFactTimeStamp(), false);
+ boolean entryAdded =
+ CarbonLoaderUtil.recordNewLoadMetadata(newLoadMetaEntry, model, true, insertOverwrite);
+ if (!entryAdded) {
+ throw new IOException("Failed to add entry in table status for " + model.getTableName());
+ }
+ }
+
+ /**
+ * This method will update the load failure entry in the table status file
+ */
+ public static void updateTableStatusForFailure(CarbonLoadModel model)
+ throws IOException {
+ // in case if failure the load status should be "Marked for delete" so that it will be taken
+ // care during clean up
+ SegmentStatus loadStatus = SegmentStatus.MARKED_FOR_DELETE;
+ // always the last entry in the load metadata details will be the current load entry
+ LoadMetadataDetails loadMetaEntry = model.getCurrentLoadMetadataDetail();
+ CarbonLoaderUtil
+ .populateNewLoadMetaEntry(loadMetaEntry, loadStatus, model.getFactTimeStamp(), true);
+ boolean entryAdded =
+ CarbonLoaderUtil.recordNewLoadMetadata(loadMetaEntry, model, false, false);
+ if (!entryAdded) {
+ throw new IOException(
+ "Failed to update failure entry in table status for " + model.getTableName());
+ }
+ }
+
public static Dictionary getDictionary(DictionaryColumnUniqueIdentifier columnIdentifier)
throws IOException {
Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache =
@@ -827,4 +857,18 @@ public final class CarbonLoaderUtil {
return newListMetadata;
}
+ /*
+ * This method will add data size and index size into tablestatus for each segment
+ */
+ public static void addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
+ String segmentId, CarbonTable carbonTable) throws IOException {
+ CarbonTablePath carbonTablePath =
+ CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
+ Map<String, Long> dataIndexSize =
+ CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
+ loadMetadataDetails
+ .setDataSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString());
+ loadMetadataDetails
+ .setIndexSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).toString());
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91e6f6f4/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 8c4d5ba..37aaea5 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -315,7 +315,7 @@ object StreamHandoffRDD {
}
if (loadStatus == SegmentStatus.LOAD_FAILURE) {
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.info("********starting clean up**********")
CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
LOGGER.info("********clean up done**********")
[38/50] [abbrv] carbondata git commit: [CARBONDATA-1912] Handling
lock issues
Posted by ja...@apache.org.
[CARBONDATA-1912] Handling lock issues
Description :
Getting error trace in spark-sql console while executing compaction and alter table rename commands.
Scenario:
execute compaction and alter table rename queries in spark - sql
Even though operation is success , getting error trace on the console
Solution: In compaction we are already handling unlock after operation , so we can handle unlock in catch block instead of finally block.
In alter table rename operation: no need to unlock file on old location which will result error due to table name change.
This closes #1684
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fc93de16
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fc93de16
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fc93de16
Branch: refs/heads/fgdatamap
Commit: fc93de163f472db92243fdfd5ef119765da97237
Parents: a8cbacf
Author: Manohar <ma...@gmail.com>
Authored: Tue Dec 19 19:12:14 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Dec 22 10:05:26 2017 +0530
----------------------------------------------------------------------
.../command/management/CarbonAlterTableCompactionCommand.scala | 5 ++---
.../command/schema/CarbonAlterTableRenameCommand.scala | 4 ++--
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc93de16/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 517fbda..45d3537 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -227,7 +227,7 @@ case class CarbonAlterTableCompactionCommand(
"")
OperationListenerBus.getInstance
.fireEvent(alterTableCompactionPreStatusUpdateEvent, operationContext)
-
+ lock.unlock()
} else {
CarbonDataRDDFactory.startCompactionThreads(
sqlContext,
@@ -241,9 +241,8 @@ case class CarbonAlterTableCompactionCommand(
} catch {
case e: Exception =>
LOGGER.error(s"Exception in start compaction thread. ${ e.getMessage }")
+ lock.unlock()
throw e
- } finally {
- lock.unlock()
}
} else {
LOGGER.audit("Not able to acquire the compaction lock for table " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc93de16/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 1766064..28ed581 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -166,10 +166,10 @@ private[sql] case class CarbonAlterTableRenameCommand(
sparkSession)
renameBadRecords(newTableName, oldTableName, oldDatabaseName)
}
+ // release lock from old location in case of any rename failure
+ AlterTableUtil.releaseLocks(locks)
sys.error(s"Alter table rename table operation failed: ${e.getMessage}")
} finally {
- // release lock after command execution completion
- AlterTableUtil.releaseLocks(locks)
// case specific to rename table as after table rename old table path will not be found
if (carbonTable != null) {
val newTablePath = CarbonUtil
[37/50] [abbrv] carbondata git commit: [CARBONDATA-1618] SDV test
cases for Table comment support for alter table
Posted by ja...@apache.org.
[CARBONDATA-1618] SDV test cases for Table comment support for alter table
This closes #1612
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a8cbacf3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a8cbacf3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a8cbacf3
Branch: refs/heads/fgdatamap
Commit: a8cbacf3e923600d509fc080ebef6a4eaf992c3e
Parents: e2360c7
Author: Pawan Malwal <pm...@gmail.com>
Authored: Tue Dec 5 12:55:46 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Dec 22 09:58:27 2017 +0530
----------------------------------------------------------------------
.../TableCommentAlterTableTestCase.scala | 146 +++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 4 +-
2 files changed, 149 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8cbacf3/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
new file mode 100644
index 0000000..8bd53ae
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TableCommentAlterTableTestCase.scala
@@ -0,0 +1,146 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for AlterTableTestCase to verify all scenerios
+ */
+
+class TableCommentAlterTableTestCase extends QueryTest with BeforeAndAfterAll {
+
+
+ //Check create table with table comment
+ test("TableCommentAlterTable_001_01", Include) {
+ sql(s"""drop table if exists table_comment""").collect
+ sql(s""" create table table_comment(id int, name string) comment "This is table comment" STORED BY 'carbondata'""").collect
+ val result = sql("describe formatted table_comment")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "This is table comment")
+ }
+
+ //Check create table with comment as numeric/alphanumeric/special symbols
+ test("TableCommentAlterTable_001_02", Include) {
+ sql(s"""drop table if exists table_comment_special""").collect
+ sql(s"""create table table_comment_special(id int, name string) comment "3432432 @This $$$$ is table comment #$$%^&*@# 34234" STORED BY 'carbondata'""").collect
+ val result = sql("describe formatted table_comment_special")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "3432432 @This $$ is table comment #$%^&*@# 34234")
+ }
+
+ //Check create table with comment as blank
+ test("TableCommentAlterTable_001_03", Include) {
+ sql(s"""drop table if exists table_comment_alphabets""").collect
+ sql(s"""create table table_comment_alphabets(id int, name string) comment "" STORED BY 'carbondata'""").collect
+ val result = sql("describe formatted table_comment")
+ checkExistence(result, true, "Comment")
+ }
+
+ //Check create table with comment as high values
+ test("TableCommentAlterTable_001_04", Include) {
+ sql(s"""drop table if exists table_comment_high""").collect
+ sql(s"""create table table_comment_high(id int, name string) comment "This is table comment jgfhdsjgfhsjdgjfsdgfbcfgr763 23x4bt23n8z30bt20t49ct4cb3g93t53945ctbugebgec4c3trbcu4grehrvsgyabrgyuwarcnrwbrcwe rwergrburygwae rwaeuigrwawewrcargw7aer wr766tznzYQEqoie weqeqewqeqweqweqweewqeqwe" STORED BY 'carbondata'""").collect
+ val result = sql("describe formatted table_comment_high")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "This is table comment jgfhdsjgfhsjdgjfsdgfbcfgr763 23x4bt23n8z30bt20t49ct4cb3g93t53945ctbugebgec4c3trbcu4grehrvsgyabrgyuwarcnrwbrcwe rwergrburygwae rwaeuigrwawewrcargw7aer wr766tznzYQEqoie weqeqewqeqweqweqweewqeqwe")
+ }
+
+ //Check create table with comment in the table properties
+ test("TableCommentAlterTable_001_05", Include) {
+ sql(s"""drop table if exists table_comment_asprop""").collect
+ sql(s"""create table table_comment_asprop (imei string,num bigint,game double,date timestamp,Profit decimal(3,2)) STORED BY 'carbondata' TBLPROPERTIES('comment'='This is table comment..!!')""").collect
+ val result = sql("describe formatted table_comment")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, false, "This is table comment..!!")
+ }
+
+ //Check create table with comment after stored by clause
+ test("TableCommentAlterTable_001_06", Include) {
+ intercept[AnalysisException] {
+ sql("create table table_comment_afterstoredby (id int, name string) STORED BY 'carbondata' comment 'This is table comment'")
+ }
+ }
+
+ //Check the comment by "describe formatted"
+ test("TableCommentAlterTable_001_07", Include) {
+ sql(s"""drop table if exists table_comment""").collect
+ sql(s""" create table table_comment(id int, name string) comment "This is table comment" STORED BY 'carbondata'""").collect
+ val result = sql("describe formatted table_comment")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "This is table comment")
+ }
+
+ //Check SET Command by using alter command
+ test("TableCommentAlterTable_001_08", Include) {
+ sql(s"""drop table if exists table_comment_set""").collect
+ sql(s"""create table table_comment_set (id int, name string) STORED BY 'carbondata'""").collect
+ sql(s"""alter table table_comment_set SET TBLPROPERTIES ('comment'='This table comment is SET by alter table set')""").collect
+ val result = sql("describe formatted table_comment_set")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "This table comment is SET by alter table set")
+ }
+
+ //Check UNSET Command by using alter command
+ test("TableCommentAlterTable_001_09", Include) {
+ sql(s"""drop table if exists table_comment_unset""").collect
+ sql(s"""create table table_comment_unset(id int, name string) comment "This is table comment" STORED BY 'carbondata'""").collect
+ val result1 = sql("describe formatted table_comment_unset")
+ checkExistence(result1, true, "Comment")
+ checkExistence(result1, true, "This is table comment")
+
+ sql(s"""alter table table_comment_unset UNSET TBLPROPERTIES IF EXISTS ('comment')""").collect
+ val result2 = sql("describe formatted table_comment_unset")
+ checkExistence(result2, true, "Comment")
+ checkExistence(result2, false, "This is table comment")
+ }
+
+ //Check RENAME by using alter command
+ test("TableCommentAlterTable_001_10", Include) {
+ sql(s"""drop table if exists table_comment_rename1""").collect
+ sql(s"""create table table_comment_rename1 (id int, name string) STORED BY 'carbondata'""").collect
+ sql(s"""alter table table_comment_rename1 rename to table_comment_rename2""").collect
+ val result = sql("describe formatted table_comment_rename2")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "table_comment")
+ }
+
+ //Check ADD Columns by using alter command
+ test("TableCommentAlterTable_001_11", Include) {
+ sql(s"""drop table if exists table_add_column """).collect
+ sql(s"""create table table_add_column (name string, country string, upd_time timestamp) stored by 'carbondata'""").collect
+ sql(s"""alter table table_add_column add columns (id bigint)""").collect
+ val result = sql("describe table_add_column ")
+ checkExistence(result, true, "id")
+ checkExistence(result, true, "bigint")
+ }
+
+ //Check CHANGE DATA TYPE by using alter command
+ test("TableCommentAlterTable_001_12", Include) {
+ sql(s"""drop table if exists table_change_datatype""").collect
+ sql(s"""create table table_change_datatype (name string, id decimal(3,2),country string) stored by 'carbondata'""").collect
+ sql(s"""alter table table_change_datatype change id id decimal(10,4)""").collect
+ val result = sql("describe formatted table_change_datatype")
+ checkExistence(result, true, "Comment")
+ checkExistence(result, true, "decimal(10,4)")
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8cbacf3/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index 440d333..b968ae6 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -52,6 +52,7 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
new ShowLoadsTestCase ::
new SinglepassTestCase ::
new SortColumnTestCase ::
+ new TableCommentAlterTableTestCase ::
new TimestamptypesTestCase ::
new V3offheapvectorTestCase ::
new Vector1TestCase ::
@@ -135,7 +136,8 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
new QueriesSparkBlockDistTestCase ::
new DataLoadingV3TestCase ::
new QueriesIncludeDictionaryTestCase ::
- new TestRegisterCarbonTable :: Nil
+ new TestRegisterCarbonTable ::
+ new TableCommentAlterTableTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
[32/50] [abbrv] carbondata git commit: [CARBONDATA-1908][PARTITION]
Support UPDATE/DELETE on partition tables.
Posted by ja...@apache.org.
[CARBONDATA-1908][PARTITION] Support UPDATE/DELETE on partition tables.
This closes #1681
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a89587e7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a89587e7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a89587e7
Branch: refs/heads/fgdatamap
Commit: a89587e70dddd55b44050423810776edaad26f0b
Parents: f1c6ddd
Author: ravipesala <ra...@gmail.com>
Authored: Tue Dec 19 17:43:00 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Dec 21 16:39:09 2017 +0530
----------------------------------------------------------------------
.../hadoop/api/CarbonOutputCommitter.java | 30 +++-
.../hadoop/api/CarbonTableInputFormat.java | 3 +-
.../hadoop/api/CarbonTableOutputFormat.java | 13 +-
.../iud/DeleteCarbonTableTestCase.scala | 21 +++
.../iud/UpdateCarbonTableTestCase.scala | 68 +++++++++
.../carbondata/spark/util/CommonUtil.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 23 ++-
...rbonAlterTableDropHivePartitionCommand.scala | 150 +++++++++++++++++++
.../CarbonStandardAlterTableDropPartition.scala | 150 -------------------
.../datasources/CarbonFileFormat.scala | 9 +-
.../sql/execution/strategy/DDLStrategy.scala | 4 +-
.../loading/CarbonDataLoadConfiguration.java | 11 ++
.../loading/DataLoadProcessBuilder.java | 6 +-
.../iterator/CarbonOutputIteratorWrapper.java | 61 +++++---
.../store/CarbonFactDataHandlerColumnar.java | 4 +
.../store/CarbonFactDataHandlerModel.java | 8 +
.../processing/util/CarbonLoaderUtil.java | 14 +-
17 files changed, 384 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 08fd1ac..6f5d0e4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -18,12 +18,18 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.metadata.PartitionMapFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
@@ -76,18 +82,32 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
super.commitJob(context);
boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
String segmentPath =
CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
// Merge all partition files into a single file.
new PartitionMapFileStore().mergePartitionMapFiles(segmentPath,
loadModel.getFactTimeStamp() + "");
- LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
CarbonLoaderUtil.populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS,
loadModel.getFactTimeStamp(), true);
- CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(),
- loadModel.getCarbonDataLoadSchema().getCarbonTable());
- CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
- new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath);
+ CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
+ long segmentSize = CarbonLoaderUtil
+ .addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
+ if (segmentSize > 0) {
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
+ new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath);
+ String updateTime =
+ context.getConfiguration().get(CarbonTableOutputFormat.UPADTE_TIMESTAMP, null);
+ if (updateTime != null) {
+ Set<String> segmentSet = new HashSet<>(
+ new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
+ .getValidAndInvalidSegments().getValidSegments());
+ CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, updateTime, true,
+ new ArrayList<String>());
+ }
+ } else {
+ CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 6a2349a..22eca9f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -128,6 +128,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
public static final String TABLE_NAME = "mapreduce.input.carboninputformat.tableName";
public static final String PARTITIONS_TO_PRUNE =
"mapreduce.input.carboninputformat.partitions.to.prune";
+ public static final String UPADTE_T =
+ "mapreduce.input.carboninputformat.partitions.to.prune";
// a cache for carbon table, it will be used in task side
private CarbonTable carbonTable;
@@ -915,7 +917,6 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @param identifier
* @return
* @throws IOException
- * @throws KeyGenException
*/
public BlockMappingVO getBlockRowCount(Job job, AbsoluteTableIdentifier identifier,
List<String> partitions) throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index f11cb35..bd70e41 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -40,6 +40,8 @@ import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWra
import org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
@@ -87,6 +89,13 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
"mapreduce.carbontable.dict.server.host";
public static final String DICTIONARY_SERVER_PORT =
"mapreduce.carbontable.dict.server.port";
+ /**
+ * Set the update timestamp if user sets in case of update query. It needs to be updated
+ * in load status update time
+ */
+ public static final String UPADTE_TIMESTAMP = "mapreduce.carbontable.update.timestamp";
+
+ private static final Log LOG = LogFactory.getLog(CarbonTableOutputFormat.class);
private CarbonOutputCommitter committer;
@@ -373,15 +382,17 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
}
@Override public void close(TaskAttemptContext taskAttemptContext) throws InterruptedException {
- iteratorWrapper.close();
+ iteratorWrapper.closeWriter();
try {
future.get();
} catch (ExecutionException e) {
+ LOG.error("Error while loading data", e);
throw new InterruptedException(e.getMessage());
} finally {
executorService.shutdownNow();
dataLoadExecutor.close();
}
+ LOG.info("Closed partition writer task " + taskAttemptContext.getTaskAttemptID());
}
public CarbonLoadModel getLoadModel() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index efbe807..deaad20 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -104,6 +104,27 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
)
}
+ test("partition delete data from carbon table with alias [where clause ]") {
+ sql("drop table if exists iud_db.dest")
+ sql("""create table iud_db.dest (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
+ sql("""delete from iud_db.dest d where d.c1 = 'a'""").show
+ checkAnswer(
+ sql("""select c2 from iud_db.dest"""),
+ Seq(Row(2), Row(3),Row(4), Row(5))
+ )
+ }
+ test("partition delete data from carbon table[where clause ]") {
+ sql("""drop table if exists iud_db.dest""")
+ sql("""create table iud_db.dest (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
+ sql("""delete from iud_db.dest where c2 = 2""").show
+ checkAnswer(
+ sql("""select c1 from iud_db.dest"""),
+ Seq(Row("a"), Row("c"), Row("d"), Row("e"))
+ )
+ }
+
test("Records more than one pagesize after delete operation ") {
sql("DROP TABLE IF EXISTS carbon2")
import sqlContext.implicits._
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
index f265e75..25a4999 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
@@ -569,6 +569,74 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.defaultValueIsPersistEnabled)
}
+ test("partition test update operation with 0 rows updation.") {
+ sql("""drop table if exists iud.zerorows_part""").show
+ sql("""create table iud.zerorows_part (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.zerorows_part""")
+ sql("""update iud.zerorows_part d set (d.c2) = (d.c2 + 1) where d.c1 = 'a'""").show()
+ sql("""update iud.zerorows_part d set (d.c2) = (d.c2 + 1) where d.c1 = 'xxx'""").show()
+ checkAnswer(
+ sql("""select c1,c2,c3,c5 from iud.zerorows_part"""),
+ Seq(Row("a",2,"aa","aaa"),Row("b",2,"bb","bbb"),Row("c",3,"cc","ccc"),Row("d",4,"dd","ddd"),Row("e",5,"ee","eee"))
+ )
+ sql("""drop table iud.zerorows_part""").show
+
+ }
+
+
+ test("partition update carbon table[select from source table with where and exist]") {
+ sql("""drop table if exists iud.dest11_part""").show
+ sql("""create table iud.dest11_part (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest11_part""")
+ sql("""update iud.dest11_part d set (d.c3, d.c5 ) = (select s.c33,s.c55 from iud.source2 s where d.c1 = s.c11) where 1 = 1""").show()
+ checkAnswer(
+ sql("""select c3,c5 from iud.dest11_part"""),
+ Seq(Row("cc","ccc"), Row("dd","ddd"),Row("ee","eee"), Row("MGM","Disco"),Row("RGK","Music"))
+ )
+ sql("""drop table iud.dest11_part""").show
+ }
+
+ test("partition update carbon table[using destination table columns with where and exist]") {
+ sql("""drop table if exists iud.dest22_part""")
+ sql("""create table iud.dest22_part (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest22_part""")
+ checkAnswer(
+ sql("""select c2 from iud.dest22_part where c1='a'"""),
+ Seq(Row(1))
+ )
+ sql("""update iud.dest22_part d set (d.c2) = (d.c2 + 1) where d.c1 = 'a'""").show()
+ checkAnswer(
+ sql("""select c2 from iud.dest22_part where c1='a'"""),
+ Seq(Row(2))
+ )
+ sql("""drop table if exists iud.dest22_part""")
+ }
+
+ test("partition update carbon table without alias in set columns") {
+ sql("""drop table if exists iud.dest33_part""")
+ sql("""create table iud.dest33_part (c2 int,c3 string,c5 string) PARTITIONED BY(c1 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33_part""")
+ sql("""update iud.dest33_part d set (c3,c5 ) = (select s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11) where d.c1 = 'a'""").show()
+ checkAnswer(
+ sql("""select c3,c5 from iud.dest33_part where c1='a'"""),
+ Seq(Row("MGM","Disco"))
+ )
+ sql("""drop table if exists iud.dest33_part""")
+ }
+
+ test("partition update carbon table without alias in set columns with mulitple loads") {
+ sql("""drop table if exists iud.dest33_part""")
+ sql("""create table iud.dest33_part (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33_part""")
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33_part""")
+ sql("""update iud.dest33_part d set (c3,c5 ) = (select s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11) where d.c1 = 'a'""").show()
+ checkAnswer(
+ sql("""select c3,c5 from iud.dest33_part where c1='a'"""),
+ Seq(Row("MGM","Disco"),Row("MGM","Disco"))
+ )
+ sql("""drop table if exists iud.dest33_part""")
+ }
+
override def afterAll {
sql("use default")
sql("drop database if exists iud cascade")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index ae30300..dba8f0e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -639,7 +639,7 @@ object CommonUtil {
def readLoadMetadataDetails(model: CarbonLoadModel): Unit = {
val metadataPath = model.getCarbonDataLoadSchema.getCarbonTable.getMetaDataFilepath
val details = SegmentStatusManager.readLoadMetadata(metadataPath)
- model.setLoadMetadataDetails(details.toList.asJava)
+ model.setLoadMetadataDetails(new util.ArrayList[LoadMetadataDetails](details.toList.asJava))
}
def configureCSVInputFormat(configuration: Configuration,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 69be362..7492951 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogRelation, CatalogTable}
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
import org.apache.spark.sql.execution.LogicalRDD
+import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY
import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
@@ -490,7 +491,24 @@ case class CarbonLoadDataCommand(
}
InternalRow.fromSeq(data)
}
- LogicalRDD(attributes, rdd)(sparkSession)
+ if (updateModel.isDefined) {
+ sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
+ // In case of update, we don't need the segmrntid column in case of partitioning
+ val dropAttributes = attributes.dropRight(1)
+ val finalOutput = relation.output.map { attr =>
+ dropAttributes.find { d =>
+ val index = d.name.lastIndexOf("-updatedColumn")
+ if (index > 0) {
+ d.name.substring(0, index).equalsIgnoreCase(attr.name)
+ } else {
+ d.name.equalsIgnoreCase(attr.name)
+ }
+ }.get
+ }
+ Project(finalOutput, LogicalRDD(attributes, rdd)(sparkSession))
+ } else {
+ LogicalRDD(attributes, rdd)(sparkSession)
+ }
} else {
var timeStampformatString = carbonLoadModel.getTimestampformat
@@ -618,6 +636,9 @@ case class CarbonLoadDataCommand(
options += (("onepass", loadModel.getUseOnePass.toString))
options += (("dicthost", loadModel.getDictionaryServerHost))
options += (("dictport", loadModel.getDictionaryServerPort.toString))
+ if (updateModel.isDefined) {
+ options += (("updatetimestamp", updateModel.get.updatedTimeStamp.toString))
+ }
val hdfsRelation = HadoopFsRelation(
location = catalog,
partitionSchema = partitionSchema,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
new file mode 100644
index 0000000..c68e43c
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.partition
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
+import org.apache.spark.util.AlterTableUtil
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.spark.rdd.{CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
+
+/**
+ * Drop the partitions from hive and carbon store. It drops the partitions in following steps
+ * 1. Drop the partitions from carbon store, it just create one new mapper file in each segment
+ * with uniqueid.
+ * 2. Drop partitions from hive.
+ * 3. In any above step fails then roll back the newly created files
+ * 4. After success of steps 1 and 2 , it commits the files by removing the old fails.
+ * Here it does not remove any data from store. During compaction the old data won't be considered.
+ * @param tableName
+ * @param specs
+ * @param ifExists
+ * @param purge
+ * @param retainData
+ */
+case class CarbonAlterTableDropHivePartitionCommand(
+ tableName: TableIdentifier,
+ specs: Seq[TablePartitionSpec],
+ ifExists: Boolean,
+ purge: Boolean,
+ retainData: Boolean)
+ extends AtomicRunnableCommand {
+
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ if (table.isHivePartitionTable) {
+ try {
+ specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
+ } catch {
+ case e: Exception =>
+ if (!ifExists) {
+ throw e
+ } else {
+ log.warn(e.getMessage)
+ return Seq.empty[Row]
+ }
+ }
+
+ // Drop the partitions from hive.
+ AlterTableDropPartitionCommand(tableName, specs, ifExists, purge, retainData)
+ .run(sparkSession)
+ }
+ Seq.empty[Row]
+ }
+
+
+ override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
+ AlterTableAddPartitionCommand(tableName, specs.map((_, None)), ifExists)
+ val msg = s"Got exception $exception when processing data of drop partition." +
+ "Adding back partitions to the metadata"
+ LogServiceFactory.getLogService(this.getClass.getCanonicalName).error(msg)
+ Seq.empty[Row]
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ var locks = List.empty[ICarbonLock]
+ val uniqueId = System.currentTimeMillis().toString
+ try {
+ val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
+ LockUsage.COMPACTION_LOCK,
+ LockUsage.DELETE_SEGMENT_LOCK,
+ LockUsage.DROP_TABLE_LOCK,
+ LockUsage.CLEAN_FILES_LOCK,
+ LockUsage.ALTER_PARTITION_LOCK)
+ locks = AlterTableUtil.validateTableAndAcquireLock(
+ table.getDatabaseName,
+ table.getTableName,
+ locksToBeAcquired)(sparkSession)
+ val partitionNames = specs.flatMap { f =>
+ f.map(k => k._1 + "=" + k._2)
+ }.toSet
+ val segments = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
+ .getValidAndInvalidSegments.getValidSegments
+ try {
+ // First drop the partitions from partition mapper files of each segment
+ new CarbonDropPartitionRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ partitionNames.toSeq,
+ uniqueId).collect()
+ } catch {
+ case e: Exception =>
+ // roll back the drop partitions from carbon store
+ new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ false,
+ uniqueId).collect()
+ throw e
+ }
+ // commit the drop partitions from carbon store
+ new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ true,
+ uniqueId).collect()
+ // Update the loadstatus with update time to clear cache from driver.
+ val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
+ .getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
+ CarbonUpdateUtil.updateTableMetadataStatus(
+ segmentSet,
+ table,
+ uniqueId,
+ true,
+ new util.ArrayList[String])
+ DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
+ } finally {
+ AlterTableUtil.releaseLocks(locks)
+ }
+ Seq.empty[Row]
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
deleted file mode 100644
index fd59587..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command.partition
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
-import org.apache.spark.util.AlterTableUtil
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.spark.rdd.{CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
-
-/**
- * Drop the partitions from hive and carbon store. It drops the partitions in following steps
- * 1. Drop the partitions from carbon store, it just create one new mapper file in each segment
- * with uniqueid.
- * 2. Drop partitions from hive.
- * 3. In any above step fails then roll back the newly created files
- * 4. After success of steps 1 and 2 , it commits the files by removing the old fails.
- * Here it does not remove any data from store. During compaction the old data won't be considered.
- * @param tableName
- * @param specs
- * @param ifExists
- * @param purge
- * @param retainData
- */
-case class CarbonStandardAlterTableDropPartition(
- tableName: TableIdentifier,
- specs: Seq[TablePartitionSpec],
- ifExists: Boolean,
- purge: Boolean,
- retainData: Boolean)
- extends AtomicRunnableCommand {
-
-
- override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- if (table.isHivePartitionTable) {
- try {
- specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
- } catch {
- case e: Exception =>
- if (!ifExists) {
- throw e
- } else {
- log.warn(e.getMessage)
- return Seq.empty[Row]
- }
- }
-
- // Drop the partitions from hive.
- AlterTableDropPartitionCommand(tableName, specs, ifExists, purge, retainData)
- .run(sparkSession)
- }
- Seq.empty[Row]
- }
-
-
- override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
- AlterTableAddPartitionCommand(tableName, specs.map((_, None)), ifExists)
- val msg = s"Got exception $exception when processing data of drop partition." +
- "Adding back partitions to the metadata"
- LogServiceFactory.getLogService(this.getClass.getCanonicalName).error(msg)
- Seq.empty[Row]
- }
-
- override def processData(sparkSession: SparkSession): Seq[Row] = {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- var locks = List.empty[ICarbonLock]
- val uniqueId = System.currentTimeMillis().toString
- try {
- val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
- LockUsage.COMPACTION_LOCK,
- LockUsage.DELETE_SEGMENT_LOCK,
- LockUsage.DROP_TABLE_LOCK,
- LockUsage.CLEAN_FILES_LOCK,
- LockUsage.ALTER_PARTITION_LOCK)
- locks = AlterTableUtil.validateTableAndAcquireLock(
- table.getDatabaseName,
- table.getTableName,
- locksToBeAcquired)(sparkSession)
- val partitionNames = specs.flatMap { f =>
- f.map(k => k._1 + "=" + k._2)
- }.toSet
- val segments = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
- .getValidAndInvalidSegments.getValidSegments
- try {
- // First drop the partitions from partition mapper files of each segment
- new CarbonDropPartitionRDD(sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- partitionNames.toSeq,
- uniqueId).collect()
- } catch {
- case e: Exception =>
- // roll back the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- false,
- uniqueId).collect()
- throw e
- }
- // commit the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- true,
- uniqueId).collect()
- // Update the loadstatus with update time to clear cache from driver.
- val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
- .getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
- CarbonUpdateUtil.updateTableMetadataStatus(
- segmentSet,
- table,
- uniqueId,
- true,
- new util.ArrayList[String])
- DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
- } finally {
- AlterTableUtil.releaseLocks(locks)
- }
- Seq.empty[Row]
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 3eed726..a95693c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -104,8 +104,15 @@ with Serializable {
model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
- CarbonTableOutputFormat.setLoadModel(conf, model)
CarbonTableOutputFormat.setOverwrite(conf, options("overwrite").toBoolean)
+ // Set the update timestamp if user sets in case of update query. It needs to be updated
+ // in load status update time
+ val updateTimeStamp = options.getOrElse("updatetimestamp", null)
+ if (updateTimeStamp != null) {
+ conf.set(CarbonTableOutputFormat.UPADTE_TIMESTAMP, updateTimeStamp)
+ model.setFactTimeStamp(updateTimeStamp.toLong)
+ }
+ CarbonTableOutputFormat.setLoadModel(conf, model)
new OutputWriterFactory {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index eeadbf6..89fcfd2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompactionCommand, CarbonInsertIntoCommand, CarbonLoadDataCommand, RefreshCarbonTableCommand}
-import org.apache.spark.sql.execution.command.partition.{CarbonShowCarbonPartitionsCommand, CarbonStandardAlterTableDropPartition}
+import org.apache.spark.sql.execution.command.partition.{CarbonAlterTableDropHivePartitionCommand, CarbonShowCarbonPartitionsCommand}
import org.apache.spark.sql.execution.command.schema._
import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedCommand, CarbonDropTableCommand}
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand}
@@ -174,7 +174,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
.tableExists(tableName)(sparkSession)
if (isCarbonTable) {
ExecutedCommandExec(
- CarbonStandardAlterTableDropPartition(
+ CarbonAlterTableDropHivePartitionCommand(
tableName,
specs,
ifExists,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 19c8f03..7b1ab9d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -104,6 +104,11 @@ public class CarbonDataLoadConfiguration {
// contains metadata used in write step of loading process
private TableSpec tableSpec;
+ /**
+ * Number of thread cores to use while writing data files
+ */
+ private short writingCoresCount;
+
public CarbonDataLoadConfiguration() {
}
@@ -351,5 +356,11 @@ public class CarbonDataLoadConfiguration {
this.tableSpec = tableSpec;
}
+ public short getWritingCoresCount() {
+ return writingCoresCount;
+ }
+ public void setWritingCoresCount(short writingCoresCount) {
+ this.writingCoresCount = writingCoresCount;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 787cb7b..87d5b97 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -224,7 +224,11 @@ public final class DataLoadProcessBuilder {
configuration.setPreFetch(loadModel.isPreFetch());
configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
-
+ // For partition loading always use single core as it already runs in multiple
+ // threads per partition
+ if (carbonTable.isHivePartitionTable()) {
+ configuration.setWritingCoresCount((short) 1);
+ }
TableSpec tableSpec = new TableSpec(dimensions, measures);
configuration.setTableSpec(tableSpec);
return configuration;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
index 08d1497..66943c8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
@@ -18,15 +18,21 @@
package org.apache.carbondata.processing.loading.iterator;
import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.CarbonIterator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* It is wrapper class to hold the rows in batches when record writer writes the data and allows
* to iterate on it during data load. It uses blocking queue to coordinate between read and write.
*/
public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
+ private static final Log LOG = LogFactory.getLog(CarbonOutputIteratorWrapper.class);
+
private boolean close = false;
/**
@@ -50,34 +56,48 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
@Override
public boolean hasNext() {
- return !queue.isEmpty() || !close || readBatch != null && readBatch.hasNext();
- }
-
- @Override
- public String[] next() {
- if (readBatch == null || !readBatch.hasNext() && !close) {
+ if (readBatch == null || !readBatch.hasNext()) {
try {
- readBatch = queue.take();
+ if (!close) {
+ readBatch = queue.poll(5, TimeUnit.MINUTES);
+ if (readBatch == null) {
+ LOG.warn("This scenario should not happen");
+ return false;
+ }
+ } else {
+ readBatch = queue.poll();
+ if (readBatch == null) {
+ return false;
+ }
+ }
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
- return readBatch.next();
+ return readBatch.hasNext();
}
@Override
- public void close() {
- if (loadBatch.isLoading()) {
- try {
- loadBatch.readyRead();
- if (loadBatch.size > 0) {
- queue.put(loadBatch);
- }
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
+ public String[] next() {
+ return readBatch.next();
+ }
+
+ public void closeWriter() {
+ try {
+ loadBatch.readyRead();
+ if (loadBatch.size > 0) {
+ queue.put(loadBatch);
}
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
}
close = true;
+ // It is required if the thread waits for take.
+ if (queue.isEmpty()) {
+ if (!queue.offer(new RowBatch(0))) {
+ LOG.warn("The default last element is not added to queue");
+ }
+ }
}
private static class RowBatch extends CarbonIterator<String[]> {
@@ -88,8 +108,6 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
private int size;
- private boolean isLoading = true;
-
private RowBatch(int size) {
batch = new String[size][];
this.size = size;
@@ -108,11 +126,6 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
public void readyRead() {
size = counter;
counter = 0;
- isLoading = false;
- }
-
- public boolean isLoading() {
- return isLoading;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 75fcea3..2e62d28 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -161,6 +161,10 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
if (sortScope != null && sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
numberOfCores = 1;
}
+ // Overriding it to the task specified cores.
+ if (model.getWritingCoresCount() > 0) {
+ numberOfCores = model.getWritingCoresCount();
+ }
blockletProcessingCount = new AtomicInteger(0);
producerExecutorService = Executors.newFixedThreadPool(numberOfCores,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 96bd2e3..d15152c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -161,6 +161,8 @@ public class CarbonFactDataHandlerModel {
private DataMapWriterListener dataMapWriterlistener;
+ private short writingCoresCount;
+
/**
* Create the model using @{@link CarbonDataLoadConfiguration}
*/
@@ -260,6 +262,7 @@ public class CarbonFactDataHandlerModel {
DataMapWriterListener listener = new DataMapWriterListener();
listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId());
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
+ carbonFactDataHandlerModel.writingCoresCount = configuration.getWritingCoresCount();
return carbonFactDataHandlerModel;
}
@@ -321,6 +324,7 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.tableSpec = new TableSpec(
segmentProperties.getDimensions(),
segmentProperties.getMeasures());
+
return carbonFactDataHandlerModel;
}
@@ -563,6 +567,10 @@ public class CarbonFactDataHandlerModel {
return sortScope;
}
+ public short getWritingCoresCount() {
+ return writingCoresCount;
+ }
+
public DataMapWriterListener getDataMapWriterlistener() {
return dataMapWriterlistener;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a89587e7/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 10fdd31..fef6930 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -858,17 +858,19 @@ public final class CarbonLoaderUtil {
}
/*
- * This method will add data size and index size into tablestatus for each segment
+ * This method will add data size and index size into tablestatus for each segment. And also
+ * returns the size of the segment.
*/
- public static void addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
+ public static Long addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
String segmentId, CarbonTable carbonTable) throws IOException {
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
Map<String, Long> dataIndexSize =
CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
- loadMetadataDetails
- .setDataSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString());
- loadMetadataDetails
- .setIndexSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).toString());
+ Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
+ loadMetadataDetails.setDataSize(String.valueOf(dataSize));
+ Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE);
+ loadMetadataDetails.setIndexSize(String.valueOf(indexSize));
+ return dataSize + indexSize;
}
}
[04/50] [abbrv] carbondata git commit: [CARBONDATA-1882] select with
group by and insertoverwrite to another carbon table
Posted by ja...@apache.org.
[CARBONDATA-1882] select with group by and insertoverwrite to another carbon table
This closes #1641
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4a7fc666
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4a7fc666
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4a7fc666
Branch: refs/heads/fgdatamap
Commit: 4a7fc666c479dc3ce9b7c569eb0bbc3311dde49b
Parents: 9a2b33e
Author: kushalsaha <ku...@gmail.com>
Authored: Mon Dec 11 21:11:21 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Sat Dec 16 22:16:28 2017 +0530
----------------------------------------------------------------------
.../sdv/generated/DataLoadingTestCase.scala | 46 +++----
.../test/resources/overwriteTable1_noRecord.csv | 1 +
.../resources/overwriteTable1_someRecord.csv | 6 +
.../test/resources/overwriteTable2_noRecord.csv | 1 +
.../resources/overwriteTable2_someRecord.csv | 4 +
.../InsertIntoCarbonTableTestCase.scala | 122 +++++++++++++++++++
.../apache/spark/sql/test/util/QueryTest.scala | 1 +
.../spark/rdd/CarbonDataRDDFactory.scala | 40 ++++--
.../management/CarbonLoadDataCommand.scala | 8 --
.../processing/util/CarbonLoaderUtil.java | 23 +++-
10 files changed, 206 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
index c8c88e2..365547e 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
@@ -227,38 +227,42 @@ class DataLoadingTestCase extends QueryTest with BeforeAndAfterAll {
//Data load-->Extra_Column_incsv
test("BadRecord_Dataload_019", Include) {
- sql(s"""CREATE TABLE exceed_column_in_Csv (CUST_NAME String,date timestamp) STORED BY 'org.apache.carbondata.format'""").collect
- intercept[Exception] {
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/extra_column.csv' into table exceed_column_in_Csv OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='REDIRECT','FILEHEADER'='CUST_NAME,date')""").collect
- checkAnswer(
- s"""select count(*) from exceed_column_in_Csv """,
- Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_019")
- }
- sql(s"""drop table exceed_column_in_Csv """).collect
+ sql(
+ s"""CREATE TABLE exceed_column_in_Csv (CUST_NAME String,date timestamp) STORED BY 'org.apache.carbondata.format'""".stripMargin).collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/extra_column.csv' into table exceed_column_in_Csv OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='REDIRECT','FILEHEADER'='CUST_NAME,date')""".stripMargin).collect
+ checkAnswer(s"""select count(*) from exceed_column_in_Csv """,Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_019")
+ sql(s"""drop table exceed_column_in_Csv """).collect
}
//Data load-->Timestamp Exceed Range
test("BadRecord_Dataload_020", Include) {
- sql(s"""CREATE TABLE timestamp_range (date timestamp) STORED BY 'org.apache.carbondata.format'""").collect
- intercept[Exception] {
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/timetsmap.csv' into table timestamp_range OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='REDIRECT','FILEHEADER'='date')""").collect
- }
- checkAnswer(s"""select count(*) from timestamp_range""",
- Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_020")
- sql(s"""drop table timestamp_range""").collect
+ sql(
+ s"""CREATE TABLE timestamp_range (date timestamp) STORED BY 'org.apache.carbondata.format'""".stripMargin).collect
+ sql(
+ s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/timetsmap.csv' into table timestamp_range OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='REDIRECT','FILEHEADER'='date')""".stripMargin).collect
+ checkAnswer(s"""select count(*) from timestamp_range""",Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_020")
+ sql(s"""drop table timestamp_range""").collect
}
//Show loads-->Delimeter_check
test("BadRecord_Dataload_021", Include) {
- sql(s"""CREATE TABLE bad_records_test5 (String_col string,integer_col int,decimal_column decimal,date timestamp,double_col double) STORED BY 'org.apache.carbondata.format'""").collect
- intercept[Exception] {
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/badrecords_test5.csv' into table bad_records_test5 OPTIONS('DELIMITER'='*' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='FALSE', 'BAD_RECORDS_ACTION'='IGNORE','FILEHEADER'='String_col,integer_col,decimal_column,date,double_col') """).collect
- }
- checkAnswer(s"""select count(*) from bad_records_test5""",
+ sql(
+ s"""CREATE TABLE bad_records_test5 (String_col string,integer_col int,decimal_column
+ |decimal,date timestamp,double_col double) STORED BY 'org.apache.carbondata.format'"""
+ .stripMargin)
+ .collect
+ sql(
+ s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/badrecords_test5.csv' into table
+ |bad_records_test5 OPTIONS('DELIMITER'='*' , 'QUOTECHAR'='"',
+ |'BAD_RECORDS_LOGGER_ENABLE'='FALSE', 'BAD_RECORDS_ACTION'='IGNORE',
+ |'FILEHEADER'='String_col,integer_col,decimal_column,date,double_col') """.stripMargin)
+ .collect
+ checkAnswer(
+ s"""select count(*) from bad_records_test5""",
Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_021")
- sql(s"""drop table bad_records_test5 """).collect
+ sql(s"""drop table bad_records_test5 """).collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-test/src/test/resources/overwriteTable1_noRecord.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/overwriteTable1_noRecord.csv b/integration/spark-common-test/src/test/resources/overwriteTable1_noRecord.csv
new file mode 100644
index 0000000..5aa5c94
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/overwriteTable1_noRecord.csv
@@ -0,0 +1 @@
+id,name,salary
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-test/src/test/resources/overwriteTable1_someRecord.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/overwriteTable1_someRecord.csv b/integration/spark-common-test/src/test/resources/overwriteTable1_someRecord.csv
new file mode 100644
index 0000000..4f02689
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/overwriteTable1_someRecord.csv
@@ -0,0 +1,6 @@
+id,name,salary
+1,hello,2300
+2,hi,2500
+3,xyz,4000
+4,xyz1,5000
+5,xyz2,6000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-test/src/test/resources/overwriteTable2_noRecord.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/overwriteTable2_noRecord.csv b/integration/spark-common-test/src/test/resources/overwriteTable2_noRecord.csv
new file mode 100644
index 0000000..a5da1cf
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/overwriteTable2_noRecord.csv
@@ -0,0 +1 @@
+id,name,salary,age
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-test/src/test/resources/overwriteTable2_someRecord.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/overwriteTable2_someRecord.csv b/integration/spark-common-test/src/test/resources/overwriteTable2_someRecord.csv
new file mode 100644
index 0000000..3798b2a
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/overwriteTable2_someRecord.csv
@@ -0,0 +1,4 @@
+id,name,salary,age
+9,abc,48,20
+10,abc1,90,21
+11,abc2,99,22
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index 137e1cb..c9c8a59 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -18,12 +18,15 @@ package org.apache.carbondata.spark.testsuite.allqueries
import java.io.File
+import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.path.CarbonStorePath
@@ -38,6 +41,8 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists THive")
sql("create table THive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions st
ring, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
sql(s"LOAD DATA local INPATH '$resourcesPath/100_olap.csv' INTO TABLE THive")
+ sql("drop table if exists OverwriteTable_t1")
+ sql("drop table if exists OverwriteTable_t2")
}
test("insert from hive") {
@@ -276,6 +281,121 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
sql("LOAD DATA INPATH '" + resourcesPath + "/100_olap.csv' overwrite INTO table TCarbonSourceOverwrite options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVe
rsion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')")
assert(rowCount == sql("select imei from TCarbonSourceOverwrite").count())
+
+ }
+
+ test("insert overwrite in group by scenario with t1 no record and t2 no record") {
+ queryExecution("overwriteTable1_noRecord.csv","overwriteTable2_noRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,sum(salary) as TotalSalary,'98' as age from OverwriteTable_t1 group by id,name,salary")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(!exists_t1)
+ assert(!exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select * from OverwriteTable_t2"),
+ Seq())
+ checkAnswer(sql("select * from OverwriteTable_t1"),
+ sql("select * from OverwriteTable_t2"))
+ }
+
+
+ test("insert overwrite in group by scenario with t1 no record and t2 some record") {
+ queryExecution("overwriteTable1_noRecord.csv","overwriteTable2_someRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,sum(salary) as TotalSalary,'98' as age from OverwriteTable_t1 group by id,name,salary")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(!exists_t1)
+ assert(!exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select * from OverwriteTable_t2"),
+ Seq())
+ checkAnswer(sql("select * from OverwriteTable_t1"),
+ sql("select * from OverwriteTable_t2"))
+ }
+
+ test("insert overwrite in group by scenario having record in both table") {
+ queryExecution("overwriteTable1_someRecord.csv","overwriteTable2_someRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,sum(salary) as TotalSalary,'98' as age from OverwriteTable_t1 group by id,name,salary")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(exists_t1)
+ assert(exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select count(*) from OverwriteTable_t1"), sql("select count(*) from OverwriteTable_t2"))
+ }
+
+ test("insert overwrite in group by scenario t1 some record and t2 no record") {
+ queryExecution("overwriteTable1_someRecord.csv","overwriteTable2_noRecord.csv")
+ sql("insert overwrite table OverwriteTable_t2 select id,name,sum(salary) as TotalSalary,'98' as age from OverwriteTable_t1 group by id,name,salary")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(exists_t1)
+ assert(exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select count(*) from OverwriteTable_t1"), sql("select count(*) from OverwriteTable_t2"))
+ }
+
+ test("insert overwrite without group by scenario t1 no record and t2 no record") {
+ queryExecution("overwriteTable1_noRecord.csv","overwriteTable2_noRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,salary as TotalSalary,'98' as age from OverwriteTable_t1")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(!exists_t1)
+ assert(!exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select * from OverwriteTable_t2"),
+ Seq())
+ checkAnswer(sql("select * from OverwriteTable_t1"),
+ sql("select * from OverwriteTable_t2"))
+ }
+
+
+ test("insert overwrite without group by scenario with t1 no record and t2 some record") {
+ queryExecution("overwriteTable1_noRecord.csv","overwriteTable2_someRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,salary as TotalSalary,'98' as age from OverwriteTable_t1")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(!exists_t1)
+ assert(!exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select * from OverwriteTable_t2"),
+ Seq())
+ checkAnswer(sql("select * from OverwriteTable_t1"),
+ sql("select * from OverwriteTable_t2"))
+ }
+
+ test("insert overwrite without group by scenario having record in both table") {
+ queryExecution("overwriteTable1_someRecord.csv","overwriteTable2_someRecord.csv")
+ sql ("insert overwrite table OverwriteTable_t2 select id,name,salary as TotalSalary,'98' as age from OverwriteTable_t1")
+ val exists_t1 = checkSegment("OverwriteTable_t1")
+ val exists_t2 = checkSegment("OverwriteTable_t2")
+ assert(exists_t1)
+ assert(exists_t2)
+ assert(sql("select * from OverwriteTable_t1").count() == sql("select * from OverwriteTable_t2").count())
+ checkAnswer(sql("select count(*) from OverwriteTable_t1"), sql("select count(*) from OverwriteTable_t2"))
+ }
+
+ private def queryExecution(csvFileName1: String , csvFileName2: String) : Unit ={
+ sql("drop table if exists OverwriteTable_t1")
+ sql("drop table if exists OverwriteTable_t2")
+ sql("create table OverwriteTable_t1(id int,name String,salary int) stored by 'carbondata' ")
+ sql("LOAD DATA INPATH '" + resourcesPath + s"/$csvFileName1' INTO table OverwriteTable_t1")
+ sql("create table OverwriteTable_t2(id int,name String,salary int,age String) stored by 'carbondata'")
+ sql("LOAD DATA INPATH '" + resourcesPath + s"/$csvFileName2' INTO table OverwriteTable_t2")
+ }
+
+
+
+ private def checkSegment(tableName: String) : Boolean ={
+ val storePath_t1 = metastoredb + s"/warehouse/${tableName.toLowerCase()}/Fact/Part0"
+ val carbonFile_t1: CarbonFile = FileFactory
+ .getCarbonFile(storePath_t1, FileFactory.getFileType(storePath_t1))
+ var exists: Boolean = carbonFile_t1.exists()
+ if (exists) {
+ val listFiles: Array[CarbonFile] = carbonFile_t1.listFiles()
+ exists = listFiles.size > 0
+ }
+ exists
}
test("test show segments after clean files for insert overwrite") {
@@ -309,6 +429,8 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS student")
sql("DROP TABLE IF EXISTS uniqdata")
sql("DROP TABLE IF EXISTS show_insert")
+ sql("drop table if exists OverwriteTable_t1")
+ sql("drop table if exists OverwriteTable_t2")
if (timeStampPropOrig != null) {
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
index c01cfef..0079d1e 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
@@ -104,6 +104,7 @@ class QueryTest extends PlanTest {
lazy val storeLocation = CarbonProperties.getInstance().
getProperty(CarbonCommonConstants.STORE_LOCATION)
val resourcesPath = TestQueryExecutor.resourcesPath
+ val metastoredb = TestQueryExecutor.metastoredb
val integrationPath = TestQueryExecutor.integrationPath
val dblocation = TestQueryExecutor.location
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 36a2e09..4933a45 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -376,7 +376,17 @@ object CarbonDataRDDFactory {
}
}
} else {
- loadStatus = SegmentStatus.LOAD_FAILURE
+ // if no value is there in data load, make load status Success
+ // and data load flow executes
+ if (dataFrame.isDefined && updateModel.isEmpty) {
+ val rdd = dataFrame.get.rdd
+ if (rdd.partitions == null || rdd.partitions.length == 0) {
+ LOGGER.warn("DataLoading finished. No data was loaded.")
+ loadStatus = SegmentStatus.SUCCESS
+ }
+ } else {
+ loadStatus = SegmentStatus.LOAD_FAILURE
+ }
}
if (loadStatus != SegmentStatus.LOAD_FAILURE &&
@@ -486,20 +496,21 @@ object CarbonDataRDDFactory {
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
throw new Exception(status(0)._2._2.errorMsg)
}
- // if segment is empty then fail the data load
+ // as no record loaded in new segment, new segment should be deleted
+ var newEntryLoadStatus =
if (!carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isChildDataMap &&
!CarbonLoaderUtil.isValidSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)) {
- // update the load entry in table status file for changing the status to marked for delete
- CommonUtil.updateTableStatusForFailure(carbonLoadModel)
- LOGGER.info("********starting clean up**********")
- CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
- LOGGER.info("********clean up done**********")
+
LOGGER.audit(s"Data load is failed for " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }" +
" as there is no data to load")
LOGGER.warn("Cannot write load metadata file as data load failed")
- throw new Exception("No Data to load")
+
+ SegmentStatus.MARKED_FOR_DELETE
+ } else {
+ loadStatus
}
+
writeDictionary(carbonLoadModel, result, writeAll = false)
val loadTablePreStatusUpdateEvent: LoadTablePreStatusUpdateEvent =
LoadTablePreStatusUpdateEvent(
@@ -508,7 +519,13 @@ object CarbonDataRDDFactory {
carbonLoadModel)
operationContext.setProperty("isOverwrite", overwriteTable)
OperationListenerBus.getInstance().fireEvent(loadTablePreStatusUpdateEvent, operationContext)
- val done = updateTableStatus(status, carbonLoadModel, loadStatus, overwriteTable)
+ val done =
+ updateTableStatus(
+ status,
+ carbonLoadModel,
+ loadStatus,
+ newEntryLoadStatus,
+ overwriteTable)
if (!done) {
CommonUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.info("********starting clean up**********")
@@ -769,17 +786,18 @@ object CarbonDataRDDFactory {
status: Array[(String, (LoadMetadataDetails, ExecutionErrors))],
carbonLoadModel: CarbonLoadModel,
loadStatus: SegmentStatus,
+ newEntryLoadStatus: SegmentStatus,
overwriteTable: Boolean
): Boolean = {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val metadataDetails = if (status != null && status(0) != null) {
+ val metadataDetails = if (status != null && status.size > 0 && status(0) != null) {
status(0)._2._1
} else {
new LoadMetadataDetails
}
CarbonLoaderUtil.populateNewLoadMetaEntry(
metadataDetails,
- loadStatus,
+ newEntryLoadStatus,
carbonLoadModel.getFactTimeStamp,
true)
CarbonUtil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 0affe79..0c6aeda 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -65,14 +65,6 @@ case class CarbonLoadDataCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- if (dataFrame.isDefined && updateModel.isEmpty) {
- val rdd = dataFrame.get.rdd
- if (rdd.partitions == null || rdd.partitions.length == 0) {
- LOGGER.warn("DataLoading finished. No data was loaded.")
- return Seq.empty
- }
- }
-
val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
carbonProperty.addProperty("zookeeper.enable.lock", "false")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a7fc666/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 6ac8307..b0c690b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -236,17 +236,18 @@ public final class CarbonLoaderUtil {
entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
// For insert overwrite, we will delete the old segment folder immediately
// So collect the old segments here
- String path = carbonTablePath.getCarbonDataDirectoryPath("0", entry.getLoadName());
- // add to the deletion list only if file exist else HDFS file system will throw
- // exception while deleting the file if file path does not exist
- if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
- staleFolders.add(FileFactory.getCarbonFile(path));
- }
+ addToStaleFolders(carbonTablePath, staleFolders, entry);
}
}
}
listOfLoadFolderDetails.set(indexToOverwriteNewMetaEntry, newMetaEntry);
}
+ // when no records are inserted then newSegmentEntry will be SegmentStatus.MARKED_FOR_DELETE
+ // so empty segment folder should be deleted
+ if (newMetaEntry.getSegmentStatus() == SegmentStatus.MARKED_FOR_DELETE) {
+ addToStaleFolders(carbonTablePath, staleFolders, newMetaEntry);
+ }
+
SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails
.toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
// Delete all old stale segment folders
@@ -278,6 +279,16 @@ public final class CarbonLoaderUtil {
return status;
}
+ private static void addToStaleFolders(CarbonTablePath carbonTablePath,
+ List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException {
+ String path = carbonTablePath.getCarbonDataDirectoryPath("0", entry.getLoadName());
+ // add to the deletion list only if file exist else HDFS file system will throw
+ // exception while deleting the file if file path does not exist
+ if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
+ staleFolders.add(FileFactory.getCarbonFile(path));
+ }
+ }
+
/**
* Method to create new entry for load in table status file
*
[11/50] [abbrv] carbondata git commit: [CARBONDATA-1856][PARTITION]
Support insert/load data for partition table
Posted by ja...@apache.org.
[CARBONDATA-1856][PARTITION] Support insert/load data for partition table
Changed carbonrelation to HadoopFSRelation during load in case of the partition table.
Implement sparks Fileformat interface for carbon and use carbonoutputformat inside.
Create partition.map file inside each segment for mapping between partition and index file.
This closes #1654
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4430178c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4430178c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4430178c
Branch: refs/heads/fgdatamap
Commit: 4430178c0f66f735f5d35ec733b0479d63b15fcb
Parents: 6e224dc
Author: ravipesala <ra...@gmail.com>
Authored: Sun Dec 17 11:44:36 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Dec 19 00:30:28 2017 +0800
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 198 ++++++++++++++
.../ThriftWrapperSchemaConverterImpl.java | 8 +-
.../schema/partition/PartitionType.java | 3 +-
.../core/metadata/schema/table/CarbonTable.java | 6 +
.../core/mutate/CarbonUpdateUtil.java | 80 +-----
.../core/util/path/CarbonTablePath.java | 13 +-
.../CarbonFormatDirectoryStructureTest.java | 2 +-
format/src/main/thrift/schema.thrift | 1 +
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../hadoop/api/CarbonOutputCommitter.java | 10 +
.../hadoop/api/CarbonTableInputFormat.java | 15 +-
.../hadoop/api/CarbonTableOutputFormat.java | 57 ++--
.../streaming/CarbonStreamRecordWriter.java | 2 +-
.../StandardPartitionTableLoadingTestCase.scala | 220 +++++++++++++++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 2 +-
.../carbondata/spark/rdd/UpdateDataLoad.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 8 +-
.../spark/util/GlobalDictionaryUtil.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +
.../command/carbonTableSchemaCommon.scala | 11 +-
.../spark/util/CarbonReflectionUtils.scala | 58 +++-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
.../management/CarbonInsertIntoCommand.scala | 9 +-
.../management/CarbonLoadDataCommand.scala | 267 +++++++++++++++++--
.../table/CarbonCreateTableCommand.scala | 8 +
.../datasources/CarbonFileFormat.scala | 246 +++++++++++++++++
.../strategy/CarbonLateDecodeStrategy.scala | 1 -
.../sql/execution/strategy/DDLStrategy.scala | 4 +-
.../spark/sql/hive/CarbonAnalysisRules.scala | 69 ++++-
.../sql/hive/CarbonPreAggregateRules.scala | 56 ----
.../spark/sql/parser/CarbonSparkSqlParser.scala | 4 +-
.../iterator/CarbonOutputIteratorWrapper.java | 2 +-
.../merger/AbstractResultProcessor.java | 6 +-
.../partition/spliter/RowResultProcessor.java | 2 +-
.../store/CarbonDataFileAttributes.java | 6 +-
.../store/CarbonFactDataHandlerModel.java | 2 +-
37 files changed, 1165 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
new file mode 100644
index 0000000..4b58e75
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import com.google.gson.Gson;
+
+/**
+ * Provide read and write support for partition mapping file in each segment
+ */
+public class PartitionMapFileStore {
+
+ /**
+ * Write partitionmapp file to the segment folder with indexfilename and corresponding partitions.
+ *
+ * @param segmentPath
+ * @param taskNo
+ * @param partionNames
+ * @throws IOException
+ */
+ public void writePartitionMapFile(String segmentPath, final String taskNo,
+ List<String> partionNames) throws IOException {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ // write partition info to new file.
+ if (carbonFile.exists() && partionNames.size() > 0) {
+ CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().startsWith(taskNo) && file.getName()
+ .endsWith(CarbonTablePath.INDEX_FILE_EXT);
+ }
+ });
+ if (carbonFiles != null && carbonFiles.length > 0) {
+ PartitionMapper partitionMapper = new PartitionMapper();
+ Map<String, List<String>> partitionMap = new HashMap<>();
+ partitionMap.put(carbonFiles[0].getName(), partionNames);
+ partitionMapper.setPartitionMap(partitionMap);
+ String path = segmentPath + "/" + taskNo + CarbonTablePath.PARTITION_MAP_EXT;
+ writePartitionFile(partitionMapper, path);
+ }
+ }
+ }
+
+ private void writePartitionFile(PartitionMapper partitionMapper, String path) throws IOException {
+ AtomicFileOperations fileWrite =
+ new AtomicFileOperationsImpl(path, FileFactory.getFileType(path));
+ BufferedWriter brWriter = null;
+ DataOutputStream dataOutputStream = null;
+ Gson gsonObjectToWrite = new Gson();
+ try {
+ dataOutputStream = fileWrite.openForWrite(FileWriteOperation.OVERWRITE);
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+
+ String metadataInstance = gsonObjectToWrite.toJson(partitionMapper);
+ brWriter.write(metadataInstance);
+ } finally {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ CarbonUtil.closeStreams(brWriter);
+ fileWrite.close();
+ }
+ }
+
+ /**
+ * Merge all partition files in a segment to single file.
+ *
+ * @param segmentPath
+ * @throws IOException
+ */
+ public void mergePartitionMapFiles(String segmentPath) throws IOException {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ if (carbonFile.exists()) {
+ CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
+ }
+ });
+ if (carbonFiles != null && carbonFiles.length > 1) {
+ PartitionMapper partitionMapper = null;
+ for (CarbonFile file : carbonFiles) {
+ PartitionMapper localMapper = readPartitionMap(file.getAbsolutePath());
+ if (partitionMapper == null && localMapper != null) {
+ partitionMapper = localMapper;
+ }
+ if (localMapper != null) {
+ partitionMapper = partitionMapper.merge(localMapper);
+ }
+ }
+ if (partitionMapper != null) {
+ String path = segmentPath + "/" + "mergedpartitions" + CarbonTablePath.PARTITION_MAP_EXT;
+ writePartitionFile(partitionMapper, path);
+ for (CarbonFile file : carbonFiles) {
+ FileFactory.deleteAllCarbonFilesOfDir(file);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * This method reads the partition file
+ *
+ * @param partitionMapPath
+ * @return
+ */
+ public PartitionMapper readPartitionMap(String partitionMapPath) {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ PartitionMapper partitionMapper;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(partitionMapPath, FileFactory.getFileType(partitionMapPath));
+
+ try {
+ if (!FileFactory.isFileExist(partitionMapPath, FileFactory.getFileType(partitionMapPath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ buffReader = new BufferedReader(inStream);
+ partitionMapper = gsonObjectToRead.fromJson(buffReader, PartitionMapper.class);
+ } catch (IOException e) {
+ return null;
+ } finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+
+ return partitionMapper;
+ }
+
+ public static class PartitionMapper implements Serializable {
+
+ private static final long serialVersionUID = 3582245668420401089L;
+
+ private Map<String, List<String>> partitionMap;
+
+ public PartitionMapper merge(PartitionMapper mapper) {
+ if (this == mapper) {
+ return this;
+ }
+ if (partitionMap != null && mapper.partitionMap != null) {
+ partitionMap.putAll(mapper.partitionMap);
+ }
+ if (partitionMap == null) {
+ partitionMap = mapper.partitionMap;
+ }
+ return this;
+ }
+
+ public Map<String, List<String>> getPartitionMap() {
+ return partitionMap;
+ }
+
+ public void setPartitionMap(Map<String, List<String>> partitionMap) {
+ this.partitionMap = partitionMap;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 1316a25..5d15bf8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -229,8 +229,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return org.apache.carbondata.format.PartitionType.RANGE;
case RANGE_INTERVAL:
return org.apache.carbondata.format.PartitionType.RANGE_INTERVAL;
+ case NATIVE_HIVE:
+ return org.apache.carbondata.format.PartitionType.NATIVE_HIVE;
default:
- return org.apache.carbondata.format.PartitionType.HASH;
+ return org.apache.carbondata.format.PartitionType.NATIVE_HIVE;
}
}
@@ -559,8 +561,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return PartitionType.RANGE;
case RANGE_INTERVAL:
return PartitionType.RANGE_INTERVAL;
+ case NATIVE_HIVE:
+ return PartitionType.NATIVE_HIVE;
default:
- return PartitionType.HASH;
+ return PartitionType.NATIVE_HIVE;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
index 9a4cf4a..4013715 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
@@ -23,5 +23,6 @@ public enum PartitionType {
RANGE,
RANGE_INTERVAL,
LIST,
- HASH
+ HASH,
+ NATIVE_HIVE
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4ebc02d..7732a50 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
@@ -573,6 +574,11 @@ public class CarbonTable implements Serializable {
return null != tablePartitionMap.get(getTableName());
}
+ public boolean isHivePartitionTable() {
+ PartitionInfo partitionInfo = tablePartitionMap.get(getTableName());
+ return null != partitionInfo && partitionInfo.getPartitionType() == PartitionType.NATIVE_HIVE;
+ }
+
/**
* @return absolute table identifier
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 0c9c47a..f4566ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -18,11 +18,8 @@
package org.apache.carbondata.core.mutate;
import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -382,7 +379,7 @@ public class CarbonUpdateUtil {
return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
}
- public static int getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
+ public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
// scan all the carbondata files and get the latest task ID.
@@ -397,11 +394,11 @@ public class CarbonUpdateUtil {
return false;
}
});
- int max = 0;
+ long max = 0;
if (null != dataFiles) {
for (CarbonFile file : dataFiles) {
- int taskNumber =
- Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
+ long taskNumber =
+ Long.parseLong(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
if (taskNumber > max) {
max = taskNumber;
}
@@ -412,75 +409,6 @@ public class CarbonUpdateUtil {
}
- public static String getLatestBlockNameForSegment(String segmentId, CarbonTablePath tablePath) {
- String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
-
- // scan all the carbondata files and get the latest task ID.
- CarbonFile segment =
- FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
-
- CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- int max = 0;
- if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
- int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()));
- if (taskNumber >= max) {
- return true;
- }
- }
- return false;
- }
- });
-
- // get the latest among the data files. highest task number will be at the last.
- return dataFiles[dataFiles.length - 1].getName();
- }
-
- /**
- * This method will convert a given timestamp to long value and then to string back
- *
- * @param factTimeStamp
- * @return
- */
- public static String convertTimeStampToString(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(factTimeStamp);
- return Long.toString(dateToStr.getTime());
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- return null;
- }
- }
-
- /**
- * This method will convert a given timestamp to long value and then to string back
- *
- * @param factTimeStamp
- * @return
- */
- public static long convertTimeStampToLong(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e1) {
- LOGGER
- .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
- return 0;
- }
- }
- }
-
-
/**
* Handling of the clean up of old carbondata files, index files , delte delta,
* update status files.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 0f85b64..6be1f4c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -48,6 +48,7 @@ public class CarbonTablePath extends Path {
public static final String INDEX_FILE_EXT = ".carbonindex";
public static final String MERGE_INDEX_FILE_EXT = ".carbonindexmerge";
+ public static final String PARTITION_MAP_EXT = ".partitionmap";
private static final String STREAMING_DIR = ".streaming";
private static final String STREAMING_LOG_DIR = "log";
@@ -250,7 +251,7 @@ public class CarbonTablePath extends Path {
* @return absolute path of data file stored in carbon data format
*/
public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
- Integer taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
+ Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp);
}
@@ -324,7 +325,7 @@ public class CarbonTablePath extends Path {
return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
default:
String segmentDir = getSegmentDir(partitionId, segmentId);
- return segmentDir + File.separator + getCarbonIndexFileName(Integer.parseInt(taskId),
+ return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
Integer.parseInt(bucketNumber), batchNo, timeStamp);
}
}
@@ -353,7 +354,7 @@ public class CarbonTablePath extends Path {
* @param factUpdateTimeStamp unique identifier to identify an update
* @return gets data file name only with out path
*/
- public static String getCarbonDataFileName(Integer filePartNo, Integer taskNo, int bucketNumber,
+ public static String getCarbonDataFileName(Integer filePartNo, Long taskNo, int bucketNumber,
int batchNo, String factUpdateTimeStamp) {
return DATA_PART_PREFIX + filePartNo + "-" + taskNo + BATCH_PREFIX + batchNo + "-"
+ bucketNumber + "-" + factUpdateTimeStamp + CARBON_DATA_EXT;
@@ -366,7 +367,7 @@ public class CarbonTablePath extends Path {
* @param factUpdatedTimeStamp time stamp
* @return filename
*/
- public static String getCarbonIndexFileName(int taskNo, int bucketNumber, int batchNo,
+ public static String getCarbonIndexFileName(long taskNo, int bucketNumber, int batchNo,
String factUpdatedTimeStamp) {
return taskNo + BATCH_PREFIX + batchNo + "-" + bucketNumber + "-" + factUpdatedTimeStamp
+ INDEX_FILE_EXT;
@@ -505,8 +506,8 @@ public class CarbonTablePath extends Path {
* @param taskNo
* @return
*/
- public static int getTaskIdFromTaskNo(String taskNo) {
- return Integer.parseInt(taskNo.split(BATCH_PREFIX)[0]);
+ public static long getTaskIdFromTaskNo(String taskNo) {
+ return Long.parseLong(taskNo.split(BATCH_PREFIX)[0]);
}
public static int getBatchNoFromTaskNo(String taskNo) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index 936f87f..5549806 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -53,7 +53,7 @@ public class CarbonFormatDirectoryStructureTest {
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
assertTrue(carbonTablePath.getSortIndexFilePath("t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
- assertTrue(carbonTablePath.getCarbonDataFilePath("1", "2", 3, 4, 0, 0, "999").replace("\\", "/")
+ assertTrue(carbonTablePath.getCarbonDataFilePath("1", "2", 3, 4L, 0, 0, "999").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Fact/Part1/Segment_2/part-3-4_batchno0-0-999.carbondata"));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index fc73cfb..a924009 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -63,6 +63,7 @@ enum PartitionType{
RANGE_INTERVAL = 1;
LIST = 2;
HASH = 3;
+ NATIVE_HIVE = 4; // Uses the standard partition features of spark/hive
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 88d8341..1803a12 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -523,9 +523,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
for (Map.Entry<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> entry :
segmentIndexMap.entrySet()) {
SegmentTaskIndexStore.TaskBucketHolder taskHolder = entry.getKey();
- int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
+ long taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
if (partitionInfo != null) {
- partitionIndex = partitionIdList.indexOf(taskId);
+ partitionIndex = partitionIdList.indexOf((int)taskId);
}
// matchedPartitions variable will be null in two cases as follows
// 1. the table is not a partition table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 9bcb2be..ec42adf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -21,8 +21,10 @@ import java.io.IOException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.PartitionMapFileStore;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;
@@ -59,6 +61,10 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
CarbonTableOutputFormat.setLoadModel(context.getConfiguration(), loadModel);
}
+ @Override public void setupTask(TaskAttemptContext context) throws IOException {
+ super.setupTask(context);
+ }
+
/**
* Update the tablestatus as success after job is success
*
@@ -75,6 +81,10 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(),
loadModel.getCarbonDataLoadSchema().getCarbonTable());
CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
+ String segmentPath =
+ CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
+ // Merge all partition files into a single file.
+ new PartitionMapFileStore().mergePartitionMapFiles(segmentPath);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index c16b0aa..9e8e22d 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -46,6 +46,7 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
@@ -387,7 +388,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// prune partitions for filter query on partition table
BitSet matchedPartitions = null;
- if (partitionInfo != null) {
+ if (partitionInfo != null && partitionInfo.getPartitionType() != PartitionType.NATIVE_HIVE) {
matchedPartitions = setMatchedPartitions(null, filter, partitionInfo, null);
if (matchedPartitions != null) {
if (matchedPartitions.cardinality() == 0) {
@@ -722,21 +723,21 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
List<org.apache.carbondata.hadoop.CarbonInputSplit> resultFilterredBlocks = new ArrayList<>();
int partitionIndex = 0;
List<Integer> partitionIdList = new ArrayList<>();
- if (partitionInfo != null) {
+ if (partitionInfo != null && partitionInfo.getPartitionType() != PartitionType.NATIVE_HIVE) {
partitionIdList = partitionInfo.getPartitionIds();
}
for (ExtendedBlocklet blocklet : prunedBlocklets) {
- int partitionId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(
- CarbonTablePath.DataFileUtil.getTaskNo(blocklet.getPath().toString()));
+ long partitionId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(
+ CarbonTablePath.DataFileUtil.getTaskNo(blocklet.getPath()));
// OldPartitionIdList is only used in alter table partition command because it change
// partition info first and then read data.
// For other normal query should use newest partitionIdList
- if (partitionInfo != null) {
+ if (partitionInfo != null && partitionInfo.getPartitionType() != PartitionType.NATIVE_HIVE) {
if (oldPartitionIdList != null) {
- partitionIndex = oldPartitionIdList.indexOf(partitionId);
+ partitionIndex = oldPartitionIdList.indexOf((int)partitionId);
} else {
- partitionIndex = partitionIdList.indexOf(partitionId);
+ partitionIndex = partitionIdList.indexOf((int)partitionId);
}
}
if (partitionIndex != -1) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 9504502..f11cb35 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -19,6 +19,10 @@ package org.apache.carbondata.hadoop.api;
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -28,6 +32,7 @@ import org.apache.carbondata.core.metadata.datatype.StructType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.processing.loading.DataLoadExecutor;
import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
@@ -203,26 +208,27 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
public RecordWriter<NullWritable, StringArrayWritable> getRecordWriter(
TaskAttemptContext taskAttemptContext) throws IOException {
final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
- loadModel.setTaskNo(taskAttemptContext.getTaskAttemptID().getTaskID().getId() + "");
+ loadModel.setTaskNo(System.nanoTime() + "");
final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
final CarbonOutputIteratorWrapper iteratorWrapper = new CarbonOutputIteratorWrapper();
final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
- CarbonRecordWriter recordWriter = new CarbonRecordWriter(iteratorWrapper, dataLoadExecutor);
- new Thread() {
+ ExecutorService executorService = Executors.newFixedThreadPool(1,
+ new CarbonThreadFactory("CarbonRecordWriter:" + loadModel.getTableName()));;
+ // It should be started in new thread as the underlying iterator uses blocking queue.
+ Future future = executorService.submit(new Thread() {
@Override public void run() {
try {
- dataLoadExecutor.execute(
- loadModel,
- tempStoreLocations,
- new CarbonIterator[] { iteratorWrapper });
+ dataLoadExecutor
+ .execute(loadModel, tempStoreLocations, new CarbonIterator[] { iteratorWrapper });
} catch (Exception e) {
dataLoadExecutor.close();
throw new RuntimeException(e);
}
}
- }.start();
+ });
- return recordWriter;
+ return new CarbonRecordWriter(iteratorWrapper, dataLoadExecutor, loadModel, future,
+ executorService);
}
public static CarbonLoadModel getLoadModel(Configuration conf) throws IOException {
@@ -339,28 +345,47 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
model.setCsvHeaderColumns(columns);
}
- private static class CarbonRecordWriter extends RecordWriter<NullWritable, StringArrayWritable> {
+ public static class CarbonRecordWriter extends RecordWriter<NullWritable, StringArrayWritable> {
private CarbonOutputIteratorWrapper iteratorWrapper;
private DataLoadExecutor dataLoadExecutor;
+ private CarbonLoadModel loadModel;
+
+ private ExecutorService executorService;
+
+ private Future future;
+
public CarbonRecordWriter(CarbonOutputIteratorWrapper iteratorWrapper,
- DataLoadExecutor dataLoadExecutor) {
+ DataLoadExecutor dataLoadExecutor, CarbonLoadModel loadModel, Future future,
+ ExecutorService executorService) {
this.iteratorWrapper = iteratorWrapper;
this.dataLoadExecutor = dataLoadExecutor;
+ this.loadModel = loadModel;
+ this.executorService = executorService;
+ this.future = future;
}
- @Override
- public void write(NullWritable aVoid, StringArrayWritable strings)
+ @Override public void write(NullWritable aVoid, StringArrayWritable strings)
throws InterruptedException {
iteratorWrapper.write(strings.get());
}
- @Override
- public void close(TaskAttemptContext taskAttemptContext) {
+ @Override public void close(TaskAttemptContext taskAttemptContext) throws InterruptedException {
iteratorWrapper.close();
- dataLoadExecutor.close();
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ throw new InterruptedException(e.getMessage());
+ } finally {
+ executorService.shutdownNow();
+ dataLoadExecutor.close();
+ }
+ }
+
+ public CarbonLoadModel getLoadModel() {
+ return loadModel;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index cb601d7..bad2f44 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -119,7 +119,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
String segmentId = CarbonStreamOutputFormat.getSegmentId(hadoopConf);
carbonLoadModel.setSegmentId(segmentId);
carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
- int taskNo = TaskID.forName(hadoopConf.get("mapred.tip.id")).getId();
+ long taskNo = TaskID.forName(hadoopConf.get("mapred.tip.id")).getId();
carbonLoadModel.setTaskNo("" + taskNo);
configuration = DataLoadProcessBuilder.createConfiguration(carbonLoadModel);
maxRowNums = hadoopConf.getInt(CarbonStreamOutputFormat.CARBON_STREAM_BLOCKLET_ROW_NUMS,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
new file mode 100644
index 0000000..11e95d8
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql(
+ """
+ | CREATE TABLE originMultiLoads (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
+ carbonTable.getTablePath)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".partitionmap")
+ }
+ })
+ assert(dataFiles.length == partitions)
+ }
+
+ test("data loading for partition table for one partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_partitionone", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for partition table for two partition column") {
+ sql(
+ """
+ | CREATE TABLE partitiontwo (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (doj Timestamp, empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwo OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_partitiontwo", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiontwo order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for partition table for three partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionthree (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_partitionthree", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionthree order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+ }
+
+
+ test("multiple data loading for partition table for three partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionmultiplethree (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,designation,deptname')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_partitionmultiplethree", "1", 1)
+ validateDataFiles("default_partitionmultiplethree", "2", 1)
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmultiplethree order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
+ }
+
+ test("insert data for partition table for three partition column") {
+ sql(
+ """
+ | CREATE TABLE insertpartitionthree (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into insertpartitionthree select empno,doj,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ sql(s"""insert into insertpartitionthree select empno,doj,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+ sql(s"""insert into insertpartitionthree select empno,doj,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
+
+ validateDataFiles("default_insertpartitionthree", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from insertpartitionthree order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
+ }
+
+ test("data loading for partition table for one static partition column") {
+ sql(
+ """
+ | CREATE TABLE staticpartitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into staticpartitionone PARTITION(empno='1') select empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary from originTable""")
+
+ validateDataFiles("default_staticpartitionone", "0", 1)
+ }
+
+ test("single pass loading for partition table for one partition column") {
+ sql(
+ """
+ | CREATE TABLE singlepasspartitionone (empname String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE singlepasspartitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'SINGLE_PASS'='true')""")
+
+ validateDataFiles("default_singlepasspartitionone", "0", 1)
+ }
+
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitiontwo")
+ sql("drop table if exists partitionthree")
+ sql("drop table if exists partitionmultiplethree")
+ sql("drop table if exists insertpartitionthree")
+ sql("drop table if exists staticpartitionone")
+ sql("drop table if exists singlepasspartitionone")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index b27521a..b5a9315 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -118,7 +118,7 @@ class CarbonTableSplitPartition(rddId: Int, val idx: Int, @transient val tableSp
}
class SparkPartitionLoader(model: CarbonLoadModel,
- splitIndex: Int,
+ splitIndex: Long,
storePath: String,
loadMetadataDetails: LoadMetadataDetails) {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
index 4934cbc..b6dd6b8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateDataLoad.scala
@@ -35,7 +35,7 @@ object UpdateDataLoad {
def DataLoadForUpdate(
segId: String,
- index: Int,
+ index: Long,
iter: Iterator[Row],
carbonLoadModel: CarbonLoadModel,
loadMetadataDetails: LoadMetadataDetails): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index ab47532..d8ce8f3 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -186,7 +186,7 @@ object CommonUtil {
val listInfo = tableProperties.get(CarbonCommonConstants.LIST_INFO)
if (partitionType.isEmpty) {
- isValid = false
+ isValid = true
} else {
partitionType.get.toUpperCase() match {
case "HASH" => if (!numPartitions.isDefined
@@ -207,10 +207,12 @@ object CommonUtil {
isValid &= validateTypeConvert(partitionerFields(0), _))
}
case "RANGE_INTERVAL" => isValid = false
- case _ => isValid = false
+ case _ => isValid = true
}
// only support one partition column for now
- if (partitionerFields.length > 1) isValid = false
+ if (partitionerFields.length > 1 && !partitionType.get.toUpperCase.equals("NATIVE_HIVE")) {
+ isValid = false
+ }
}
isValid
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 330582f..5f44e43 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -732,7 +732,7 @@ object GlobalDictionaryUtil {
headers
}
- if (headers.length > headerOfInputData.length) {
+ if (headers.length > headerOfInputData.length && !carbonTable.isHivePartitionTable) {
val msg = "The number of columns in the file header do not match the " +
"number of columns in the data file; Either delimiter " +
"or fileheader provided is not correct"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 1c0e58b..a39465f 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -419,6 +419,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
case "LIST" => partitionInfo = new PartitionInfo(cols.asJava, PartitionType.LIST)
partitionInfo.setListInfo(listInfo.map(_.asJava).asJava)
partitionInfo.initialize(listInfo.size + 1)
+ case _ => partitionInfo = new PartitionInfo(cols.asJava, PartitionType.NATIVE_HIVE)
+ partitionInfo.setListInfo(listInfo.map(_.asJava).asJava)
}
Some(partitionInfo)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 5b93907..089b60e 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -642,11 +642,14 @@ class TableNewProcessor(cm: TableModel) {
}
if (cm.partitionInfo.isDefined) {
val partitionInfo = cm.partitionInfo.get
- val PartitionColumnSchema = partitionInfo.getColumnSchemaList.asScala
+ val partitionColumnSchema = partitionInfo.getColumnSchemaList.asScala
val partitionCols = allColumns.filter { column =>
- PartitionColumnSchema.exists(_.getColumnName.equalsIgnoreCase(column.getColumnName))
- }.asJava
- partitionInfo.setColumnSchemaList(partitionCols)
+ partitionColumnSchema.exists(_.getColumnName.equalsIgnoreCase(column.getColumnName))
+ }
+ val orderCols =
+ partitionColumnSchema.map(
+ f => partitionCols.find(_.getColumnName.equalsIgnoreCase(f.getColumnName)).get).asJava
+ partitionInfo.setColumnSchemaList(orderCols)
tableSchema.setPartitionInfo(partitionInfo)
}
tableSchema.setTableName(cm.tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
index 6cc7ae1..ba51077 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
@@ -25,9 +25,12 @@ import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.catalyst.parser.AstBuilder
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.sources.BaseRelation
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -94,6 +97,57 @@ object CarbonReflectionUtils {
}
}
+ def getInsertIntoCommand(table: LogicalPlan,
+ partition: Map[String, Option[String]],
+ query: LogicalPlan,
+ overwrite: Boolean,
+ ifPartitionNotExists: Boolean): InsertIntoTable = {
+ val className = "org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable"
+ if (SPARK_VERSION.startsWith("2.1")) {
+ val overwriteOptions = createObject(
+ "org.apache.spark.sql.catalyst.plans.logical.OverwriteOptions",
+ overwrite.asInstanceOf[Object], Map.empty.asInstanceOf[Object])._1.asInstanceOf[Object]
+ createObject(
+ className,
+ table,
+ partition,
+ query,
+ overwriteOptions,
+ ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]
+ } else if (SPARK_VERSION.startsWith("2.2")) {
+ createObject(
+ className,
+ table,
+ partition,
+ query,
+ overwrite.asInstanceOf[Object],
+ ifPartitionNotExists.asInstanceOf[Object])._1.asInstanceOf[InsertIntoTable]
+ } else {
+ throw new UnsupportedOperationException("Unsupported Spark version")
+ }
+ }
+
+ def getLogicalRelation(relation: BaseRelation,
+ expectedOutputAttributes: Seq[Attribute],
+ catalogTable: Option[CatalogTable]): LogicalRelation = {
+ val className = "org.apache.spark.sql.execution.datasources.LogicalRelation"
+ if (SPARK_VERSION.startsWith("2.1")) {
+ createObject(
+ className,
+ relation,
+ Some(expectedOutputAttributes),
+ catalogTable)._1.asInstanceOf[LogicalRelation]
+ } else if (SPARK_VERSION.startsWith("2.2")) {
+ createObject(
+ className,
+ relation,
+ expectedOutputAttributes,
+ catalogTable)._1.asInstanceOf[LogicalRelation]
+ } else {
+ throw new UnsupportedOperationException("Unsupported Spark version")
+ }
+ }
+
def getOverWriteOption[T: TypeTag : reflect.ClassTag](name: String, obj: T): Boolean = {
var overwriteboolean: Boolean = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index d6360a7..29712de 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -623,7 +623,7 @@ object CarbonDataRDDFactory {
carbonLoadModel: CarbonLoadModel,
updateModel: Option[UpdateTableModel],
key: String,
- taskNo: Int,
+ taskNo: Long,
iter: Iterator[Row]
): Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] = {
val rddResult = new updateResultImpl()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index e5de052..19b96a8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -98,7 +98,7 @@ case class CarbonDatasourceHadoopRelation(
CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS)
}
if (data.logicalPlan.output.size >= carbonRelation.output.size) {
- CarbonInsertIntoCommand(this, data.logicalPlan, overwrite).run(sparkSession)
+ CarbonInsertIntoCommand(this, data.logicalPlan, overwrite, Map.empty).run(sparkSession)
} else {
CarbonException.analysisException(
"Cannot insert into target table because number of columns mismatch")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index ef16955..5142ef2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -26,7 +26,8 @@ import org.apache.carbondata.spark.util.CarbonSparkUtil
case class CarbonInsertIntoCommand(
relation: CarbonDatasourceHadoopRelation,
child: LogicalPlan,
- overwrite: Boolean)
+ overwrite: Boolean,
+ partition: Map[String, Option[String]])
extends DataCommand {
override def processData(sparkSession: SparkSession): Seq[Row] = {
@@ -40,7 +41,11 @@ case class CarbonInsertIntoCommand(
scala.collection.immutable.Map("fileheader" -> header),
overwrite,
null,
- Some(df)).run(sparkSession)
+ Some(df),
+ None,
+ None,
+ Map.empty,
+ partition).run(sparkSession)
// updating relation metadata. This is in case of auto detect high cardinality
relation.carbonRelation.metaData =
CarbonSparkUtil.createSparkMeta(relation.carbonRelation.carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 2afd040..c94f698 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -17,17 +17,31 @@
package org.apache.spark.sql.execution.command.management
+import java.text.SimpleDateFormat
+
import scala.collection.JavaConverters._
+import scala.collection.mutable
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.io.NullWritable
+import org.apache.hadoop.mapred.JobConf
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.rdd.NewHadoopRDD
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedAttribute}
+import org.apache.spark.sql.catalyst.catalog.{CatalogRelation, CatalogTable}
+import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
+import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
+import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.util.{CausedBy, FileUtils}
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
+import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
@@ -45,6 +59,7 @@ import org.apache.carbondata.events.{LoadTablePostExecutionEvent, LoadTablePreEx
import org.apache.carbondata.format
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
+import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
@@ -65,7 +80,8 @@ case class CarbonLoadDataCommand(
dataFrame: Option[DataFrame] = None,
updateModel: Option[UpdateTableModel] = None,
var tableInfoOp: Option[TableInfo] = None,
- internalOptions: Map[String, String] = Map.empty) extends DataCommand {
+ internalOptions: Map[String, String] = Map.empty,
+ partition: Map[String, Option[String]] = Map.empty) extends DataCommand {
override def processData(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
@@ -156,7 +172,7 @@ case class CarbonLoadDataCommand(
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
// add the start entry for the new load in the table status file
- if (updateModel.isEmpty) {
+ if (updateModel.isEmpty && !table.isHivePartitionTable) {
CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(carbonLoadModel, isOverwriteTable)
}
if (isOverwriteTable) {
@@ -217,9 +233,9 @@ case class CarbonLoadDataCommand(
LOGGER.error(ex, s"Dataload failure for $dbName.$tableName")
throw new RuntimeException(s"Dataload failure for $dbName.$tableName, ${ex.getMessage}")
case ex: Exception =>
+ LOGGER.error(ex)
// update the load entry in table status file for changing the status to marked for delete
CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
- LOGGER.error(ex)
LOGGER.audit(s"Dataload failure for $dbName.$tableName. Please check the logs")
throw ex
} finally {
@@ -344,16 +360,35 @@ case class CarbonLoadDataCommand(
} else {
dataFrame
}
- CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
- carbonLoadModel,
- columnar,
- partitionStatus,
- server,
- isOverwriteTable,
- hadoopConf,
- loadDataFrame,
- updateModel,
- operationContext)
+
+ if (carbonTable.isHivePartitionTable) {
+ try {
+ loadStandardPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
+ } finally {
+ server match {
+ case Some(dictServer) =>
+ try {
+ dictServer.writeTableDictionary(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ .getCarbonTableIdentifier.getTableId)
+ } catch {
+ case _: Exception =>
+ throw new Exception("Dataload failed due to error while writing dictionary file!")
+ }
+ case _ =>
+ }
+ }
+ } else {
+ CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
+ carbonLoadModel,
+ columnar,
+ partitionStatus,
+ server,
+ isOverwriteTable,
+ hadoopConf,
+ loadDataFrame,
+ updateModel,
+ operationContext)
+ }
}
private def loadData(
@@ -373,23 +408,203 @@ case class CarbonLoadDataCommand(
} else {
(dataFrame, dataFrame)
}
- if (!carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isChildDataMap) {
+ val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ if (!table.isChildDataMap) {
GlobalDictionaryUtil.generateGlobalDictionary(
sparkSession.sqlContext,
carbonLoadModel,
hadoopConf,
dictionaryDataFrame)
}
- CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
- carbonLoadModel,
- columnar,
- partitionStatus,
- None,
- isOverwriteTable,
- hadoopConf,
- loadDataFrame,
- updateModel,
- operationContext)
+ if (table.isHivePartitionTable) {
+ loadStandardPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
+ } else {
+ CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
+ carbonLoadModel,
+ columnar,
+ partitionStatus,
+ None,
+ isOverwriteTable,
+ hadoopConf,
+ loadDataFrame,
+ updateModel,
+ operationContext)
+ }
+ }
+
+ private def loadStandardPartition(sparkSession: SparkSession,
+ carbonLoadModel: CarbonLoadModel,
+ hadoopConf: Configuration,
+ dataFrame: Option[DataFrame]) = {
+ val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val logicalPlan =
+ sparkSession.sessionState.catalog.lookupRelation(
+ TableIdentifier(table.getTableName, Some(table.getDatabaseName)))
+ val relation = logicalPlan.collect {
+ case l: LogicalRelation => l
+ case c: CatalogRelation => c
+ }.head.asInstanceOf[LogicalPlan]
+
+ val query: LogicalPlan = if (dataFrame.isDefined) {
+ val timeStampformatString = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ val timeStampFormat = new SimpleDateFormat(timeStampformatString)
+ val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ val dateFormat = new SimpleDateFormat(dateFormatString)
+ val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
+ val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ val serializationNullFormat =
+ carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ val attributes =
+ StructType(dataFrame.get.schema.fields.map(_.copy(dataType = StringType))).toAttributes
+ val len = attributes.length
+ val rdd = dataFrame.get.rdd.map { f =>
+ val data = new Array[Any](len)
+ var i = 0
+ while (i < len) {
+ data(i) =
+ UTF8String.fromString(
+ CarbonScalaUtil.getString(f.get(i),
+ serializationNullFormat,
+ delimiterLevel1,
+ delimiterLevel2,
+ timeStampFormat,
+ dateFormat))
+ i = i + 1
+ }
+ InternalRow.fromSeq(data)
+ }
+ LogicalRDD(attributes, rdd)(sparkSession)
+
+ } else {
+ // input data from csv files. Convert to logical plan
+ CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
+ hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ val rdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
+ sparkSession.sparkContext,
+ classOf[CSVInputFormat],
+ classOf[NullWritable],
+ classOf[StringArrayWritable],
+ jobConf
+ ).map(f => InternalRow.fromSeq(f._2.get().map(UTF8String.fromString)))
+
+ val attributes =
+ StructType(carbonLoadModel.getCsvHeaderColumns.map(
+ StructField(_, StringType))).toAttributes
+ // Only select the required columns
+ Project(relation.output.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get),
+ LogicalRDD(attributes, rdd)(sparkSession))
+ }
+ val convertRelation = relation match {
+ case l: LogicalRelation =>
+ convertToLogicalRelation(l, isOverwriteTable, carbonLoadModel, sparkSession)
+ case c: CatalogRelation =>
+ convertToLogicalRelation(c, isOverwriteTable, carbonLoadModel, sparkSession)
+ }
+ Dataset.ofRows(
+ sparkSession,
+ CarbonReflectionUtils.getInsertIntoCommand(
+ convertRelation,
+ partition,
+ query,
+ isOverwriteTable,
+ false))
+ }
+
+ private def convertToLogicalRelation(
+ relation: LogicalRelation,
+ overWrite: Boolean,
+ loadModel: CarbonLoadModel,
+ sparkSession: SparkSession): LogicalRelation = {
+ val catalogTable = relation.catalogTable.get
+ val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
+ val metastoreSchema = StructType(StructType.fromAttributes(
+ relation.output).fields.map(_.copy(dataType = StringType)))
+ val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
+ val catalog = new CatalogFileIndex(
+ sparkSession, catalogTable, relation.relation.sizeInBytes)
+ if (lazyPruningEnabled) {
+ catalog
+ } else {
+ catalog.filterPartitions(Nil) // materialize all the partitions in memory
+ }
+ val partitionSchema =
+ StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(f =>
+ metastoreSchema.fields.find(_.name.equalsIgnoreCase(f.getColumnName))).map(_.get))
+
+
+ val dataSchema =
+ StructType(metastoreSchema
+ .filterNot(field => partitionSchema.contains(field.name)))
+ val options = new mutable.HashMap[String, String]()
+ options ++= catalogTable.storage.properties
+ options += (("overwrite", overWrite.toString))
+ options += (("onepass", loadModel.getUseOnePass.toString))
+ options += (("dicthost", loadModel.getDictionaryServerHost))
+ options += (("dictport", loadModel.getDictionaryServerPort.toString))
+ val hdfsRelation = HadoopFsRelation(
+ location = catalog,
+ partitionSchema = partitionSchema,
+ dataSchema = dataSchema,
+ bucketSpec = catalogTable.bucketSpec,
+ fileFormat = new CarbonFileFormat,
+ options = options.toMap)(sparkSession = sparkSession)
+
+ CarbonReflectionUtils.getLogicalRelation(hdfsRelation,
+ hdfsRelation.schema.toAttributes,
+ Some(catalogTable))
+ }
+
+ private def convertToLogicalRelation(
+ relation: CatalogRelation,
+ overWrite: Boolean,
+ loadModel: CarbonLoadModel,
+ sparkSession: SparkSession): LogicalRelation = {
+ val catalogTable =
+ CarbonReflectionUtils.getFieldOfCatalogTable("tableMeta", relation).asInstanceOf[CatalogTable]
+ val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
+ val metastoreSchema = StructType(StructType.fromAttributes(
+ relation.output).fields.map(_.copy(dataType = StringType)))
+ val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
+ // TODO nedd to find a way to avoid double lookup
+ val sizeInBytes =
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
+ catalogTable.identifier)(sparkSession).asInstanceOf[CarbonRelation].sizeInBytes
+ val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
+ if (lazyPruningEnabled) {
+ catalog
+ } else {
+ catalog.filterPartitions(Nil) // materialize all the partitions in memory
+ }
+ val partitionSchema =
+ StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(f =>
+ metastoreSchema.fields.find(_.name.equalsIgnoreCase(f.getColumnName))).map(_.get))
+
+
+ val dataSchema =
+ StructType(metastoreSchema
+ .filterNot(field => partitionSchema.contains(field.name)))
+ val options = new mutable.HashMap[String, String]()
+ options ++= catalogTable.storage.properties
+ options += (("overwrite", overWrite.toString))
+ options += (("onepass", loadModel.getUseOnePass.toString))
+ options += (("dicthost", loadModel.getDictionaryServerHost))
+ options += (("dictport", loadModel.getDictionaryServerPort.toString))
+ val hdfsRelation = HadoopFsRelation(
+ location = catalog,
+ partitionSchema = partitionSchema,
+ dataSchema = dataSchema,
+ bucketSpec = catalogTable.bucketSpec,
+ fileFormat = new CarbonFileFormat,
+ options = options.toMap)(sparkSession = sparkSession)
+
+ CarbonReflectionUtils.getLogicalRelation(hdfsRelation,
+ hdfsRelation.schema.toAttributes,
+ Some(catalogTable))
}
def getDataFrameWithTupleID(): DataFrame = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4430178c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index 314e551..a42d974 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -93,6 +93,13 @@ case class CarbonCreateTableCommand(
val carbonRelation = CarbonSparkUtil.createCarbonRelation(tableInfo, tablePath)
val rawSchema = CarbonSparkUtil.getRawSchema(carbonRelation)
sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
+ val partitionInfo = tableInfo.getFactTable.getPartitionInfo
+ val partitionString = if (partitionInfo != null) {
+ s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
+ _.getColumnName).mkString(",")})"
+ } else {
+ ""
+ }
sparkSession.sql(
s"""CREATE TABLE $dbName.$tableName
|(${ rawSchema })
@@ -103,6 +110,7 @@ case class CarbonCreateTableCommand(
| tablePath "$tablePath",
| path "$tablePath"
| $carbonSchemaString)
+ | $partitionString
""".stripMargin)
} catch {
case e: AnalysisException => throw e
[39/50] [abbrv] carbondata git commit: [CARBONDATA-1592] Added event
exceptions
Posted by ja...@apache.org.
[CARBONDATA-1592] Added event exceptions
This closes #1705
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dfe3097d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dfe3097d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dfe3097d
Branch: refs/heads/fgdatamap
Commit: dfe3097d3d507833348806fccf55f5467f916bab
Parents: fc93de1
Author: Manohar <ma...@gmail.com>
Authored: Thu Dec 21 16:42:40 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Dec 22 10:12:15 2017 +0530
----------------------------------------------------------------------
.../events/exception/EventExceptions.scala | 32 ++++++++++++++++++++
.../management/CarbonLoadDataCommand.scala | 4 +++
2 files changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfe3097d/integration/spark-common/src/main/scala/org/apache/carbondata/events/exception/EventExceptions.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/events/exception/EventExceptions.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/events/exception/EventExceptions.scala
new file mode 100644
index 0000000..4faef6f
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/events/exception/EventExceptions.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.events.exception
+
+/**
+ * for handling pre event specific exceptions
+ *
+ * @param message
+ */
+case class PreEventException(message: String, isCleanupRequired: Boolean) extends Exception(message)
+
+/**
+ * for handling post event specific exceptions
+ *
+ * @param message
+ */
+case class PostEventException(message: String) extends Exception(message)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfe3097d/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 7285d9d..5eee0bf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -62,6 +62,7 @@ import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusMan
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent, OperationContext, OperationListenerBus}
+import org.apache.carbondata.events.exception.PreEventException
import org.apache.carbondata.format
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
@@ -238,6 +239,9 @@ case class CarbonLoadDataCommand(
CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel)
LOGGER.error(ex, s"Dataload failure for $dbName.$tableName")
throw new RuntimeException(s"Dataload failure for $dbName.$tableName, ${ex.getMessage}")
+ // In case of event related exception
+ case preEventEx: PreEventException =>
+ throw new AnalysisException(preEventEx.getMessage)
case ex: Exception =>
LOGGER.error(ex)
// update the load entry in table status file for changing the status to marked for delete
[08/50] [abbrv] carbondata git commit:
[CARBONDATA-1288][DictionarySecureServer] Dictionary Secure Server
Implementation. During single pass load Dictionary key creation is done
through driver and executor communication through external ports. The commun
Posted by ja...@apache.org.
[CARBONDATA-1288][DictionarySecureServer] Dictionary Secure Server Implementation.
During single pass load Dictionary key creation is done through driver and executor communication through external ports.
The communication will happen through TCP communication which is not secure or encrypted by default.
But in case spark turn on Security parameters then carbon dictionary ports will also follows same authentication and encryption
i.e. communicate through SASL Authentication protocol and Digest-MD5 encryption.
This PR makes the dictionary server and client communication Secure and encrypted.
In case spark turn ON security and authentication through the below parameters then Carbon communication also becomes secure.
By default the communication is still non secure.
Parameters to set in spark-default.conf in order to Turn ON Secure Mode Dictionary Server Communication.
spark.authenticate true
spark.authenticate.enableSaslEncryption true
spark.authenticate.secret
Note- Turning ON this flag will turn on authentication and encryption in spark too.
This Closes #1662
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4daf0634
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4daf0634
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4daf0634
Branch: refs/heads/fgdatamap
Commit: 4daf0634e6338c627b01188a96b6d5c539308863
Parents: 91e6f6f
Author: sounak <so...@huawei.com>
Authored: Thu Jul 6 20:48:54 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Mon Dec 18 09:18:41 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 6 +
.../dictionary/client/DictionaryClient.java | 76 +------
.../client/DictionaryClientHandler.java | 126 -----------
.../client/NonSecureDictionaryClient.java | 97 +++++++++
.../NonSecureDictionaryClientHandler.java | 126 +++++++++++
.../generator/key/DictionaryMessage.java | 28 ++-
.../dictionary/server/DictionaryServer.java | 142 ++-----------
.../server/DictionaryServerHandler.java | 109 ----------
.../server/NonSecureDictionaryServer.java | 180 ++++++++++++++++
.../NonSecureDictionaryServerHandler.java | 112 ++++++++++
.../service/AbstractDictionaryServer.java | 89 ++++++++
.../service/DictionaryOnePassService.java | 30 +++
.../service/DictionaryServiceProvider.java | 27 +++
.../NonSecureDictionaryServiceProvider.java | 42 ++++
.../dictionary/client/DictionaryClientTest.java | 9 +-
.../examples/CarbonSessionExample.scala | 30 +--
.../client/SecureDictionaryClient.java | 116 ++++++++++
.../client/SecureDictionaryClientHandler.java | 104 +++++++++
.../SecureDictionaryServiceProvider.java | 34 +++
.../server/SecureDictionaryServer.java | 211 +++++++++++++++++++
.../server/SecureDictionaryServerHandler.java | 98 +++++++++
.../execution/command/carbonTableSchema.scala | 0
.../execution/command/carbonTableSchema.scala | 0
.../management/CarbonLoadDataCommand.scala | 61 ++++--
.../loading/CarbonDataLoadConfiguration.java | 42 ++++
.../loading/DataLoadProcessBuilder.java | 3 +
.../converter/impl/RowConverterImpl.java | 27 ++-
.../loading/model/CarbonLoadModel.java | 50 +++++
.../streaming/StreamSinkFactory.scala | 56 ++++-
29 files changed, 1542 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 959f4d4..f67b0c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1313,6 +1313,12 @@ public final class CarbonCommonConstants {
public static final String CARBON_LEASE_RECOVERY_RETRY_INTERVAL =
"carbon.lease.recovery.retry.interval";
+ @CarbonProperty
+ public static final String CARBON_SECURE_DICTIONARY_SERVER =
+ "carbon.secure.dictionary.server";
+
+ public static final String CARBON_SECURE_DICTIONARY_SERVER_DEFAULT = "true";
+
/**
* whether to use multi directories when loading data,
* the main purpose is to avoid single-disk-hot-spot
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
index 7910190..64e6252 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
@@ -14,82 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.carbondata.core.dictionary.client;
-import java.net.InetSocketAddress;
+package org.apache.carbondata.core.dictionary.client;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
-import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-
-/**
- * Dictionary client to connect to Dictionary server and generate dictionary values
- */
-public class DictionaryClient {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryClient.class.getName());
-
- private DictionaryClientHandler dictionaryClientHandler = new DictionaryClientHandler();
-
- private NioEventLoopGroup workerGroup;
+public interface DictionaryClient {
- /**
- * start dictionary client
- *
- * @param address
- * @param port
- */
- public void startClient(String address, int port) {
- LOGGER.audit("Starting client on " + address + " " + port);
- long start = System.currentTimeMillis();
- // Create an Event with 1 thread.
- workerGroup = new NioEventLoopGroup(1);
- Bootstrap clientBootstrap = new Bootstrap();
- clientBootstrap.group(workerGroup).channel(NioSocketChannel.class)
- .handler(new ChannelInitializer<SocketChannel>() {
- @Override public void initChannel(SocketChannel ch) throws Exception {
- ChannelPipeline pipeline = ch.pipeline();
- // Based on length provided at header, it collects all packets
- pipeline
- .addLast("LengthDecoder",
- new LengthFieldBasedFrameDecoder(1048576, 0,
- 2, 0, 2));
- pipeline.addLast("DictionaryClientHandler", dictionaryClientHandler);
- }
- });
- clientBootstrap.connect(new InetSocketAddress(address, port));
- LOGGER.info(
- "Dictionary client Started, Total time spent : " + (System.currentTimeMillis() - start));
- }
+ public void startClient(String secretKey, String address, int port, boolean encryptSecureServer);
- /**
- * for client request
- *
- * @param key
- * @return
- */
- public DictionaryMessage getDictionary(DictionaryMessage key) {
- return dictionaryClientHandler.getDictionary(key);
- }
+ public void shutDown();
- /**
- * shutdown dictionary client
- */
- public void shutDown() {
- workerGroup.shutdownGracefully();
- try {
- workerGroup.terminationFuture().sync();
- } catch (InterruptedException e) {
- LOGGER.error(e);
- }
- }
+ public DictionaryMessage getDictionary(DictionaryMessage key);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
deleted file mode 100644
index 01ef59a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.dictionary.client;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-
-/**
- * Client handler to get data.
- */
-public class DictionaryClientHandler extends ChannelInboundHandlerAdapter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryClientHandler.class.getName());
-
- private final BlockingQueue<DictionaryMessage> responseMsgQueue = new LinkedBlockingQueue<>();
-
- private ChannelHandlerContext ctx;
-
- private DictionaryChannelFutureListener channelFutureListener;
-
- @Override
- public void channelActive(ChannelHandlerContext ctx) throws Exception {
- this.ctx = ctx;
- channelFutureListener = new DictionaryChannelFutureListener(ctx);
- LOGGER.audit("Connected client " + ctx);
- super.channelActive(ctx);
- }
-
- @Override
- public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
- try {
- ByteBuf data = (ByteBuf) msg;
- DictionaryMessage key = new DictionaryMessage();
- key.readData(data);
- data.release();
- responseMsgQueue.add(key);
- } catch (Exception e) {
- LOGGER.error(e);
- throw e;
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
- LOGGER.error(cause, "exceptionCaught");
- ctx.close();
- }
-
- /**
- * client send request to server
- *
- * @param key DictionaryMessage
- * @return DictionaryMessage
- */
- public DictionaryMessage getDictionary(DictionaryMessage key) {
- DictionaryMessage dictionaryMessage;
- try {
- ByteBuf buffer = ctx.alloc().buffer();
- key.writeData(buffer);
- ctx.writeAndFlush(buffer).addListener(channelFutureListener);
- } catch (Exception e) {
- LOGGER.error(e, "Error while send request to server ");
- ctx.close();
- }
- try {
- dictionaryMessage = responseMsgQueue.poll(100, TimeUnit.SECONDS);
- if (dictionaryMessage == null) {
- StringBuilder message = new StringBuilder();
- message.append("DictionaryMessage { ColumnName: ")
- .append(key.getColumnName())
- .append(", DictionaryValue: ")
- .append(key.getDictionaryValue())
- .append(", type: ")
- .append(key.getType());
- throw new RuntimeException("Request timed out for key : " + message);
- }
- return dictionaryMessage;
- } catch (Exception e) {
- LOGGER.error(e);
- throw new RuntimeException(e);
- }
- }
-
- private static class DictionaryChannelFutureListener implements ChannelFutureListener {
-
- private ChannelHandlerContext ctx;
-
- DictionaryChannelFutureListener(ChannelHandlerContext ctx) {
- this.ctx = ctx;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- if (!future.isSuccess()) {
- LOGGER.error(future.cause(), "Error while sending request to Dictionary Server");
- ctx.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClient.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClient.java
new file mode 100644
index 0000000..cf25ee1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClient.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.dictionary.client;
+
+import java.net.InetSocketAddress;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
+
+/**
+ * Dictionary client to connect to Dictionary server and generate dictionary values
+ */
+public class NonSecureDictionaryClient implements DictionaryClient {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(NonSecureDictionaryClient.class.getName());
+
+ private NonSecureDictionaryClientHandler nonSecureDictionaryClientHandler =
+ new NonSecureDictionaryClientHandler();
+
+ private NioEventLoopGroup workerGroup;
+
+ /**
+ * start dictionary client
+ *
+ * @param address
+ * @param port
+ */
+ @Override public void startClient(String secretKey, String address, int port,
+ boolean encryptSecureServer) {
+ LOGGER.audit("Starting client on " + address + " " + port);
+ long start = System.currentTimeMillis();
+ // Create an Event with 1 thread.
+ workerGroup = new NioEventLoopGroup(1);
+ Bootstrap clientBootstrap = new Bootstrap();
+ clientBootstrap.group(workerGroup).channel(NioSocketChannel.class)
+ .handler(new ChannelInitializer<SocketChannel>() {
+ @Override public void initChannel(SocketChannel ch) throws Exception {
+ ChannelPipeline pipeline = ch.pipeline();
+ // Based on length provided at header, it collects all packets
+ pipeline
+ .addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
+ pipeline.addLast("NonSecureDictionaryClientHandler", nonSecureDictionaryClientHandler);
+ }
+ });
+ clientBootstrap.connect(new InetSocketAddress(address, port));
+ LOGGER.info(
+ "Dictionary client Started, Total time spent : " + (System.currentTimeMillis() - start));
+ }
+
+ /**
+ * for client request
+ *
+ * @param key
+ * @return
+ */
+ @Override
+ public DictionaryMessage getDictionary(
+ DictionaryMessage key) {
+ return nonSecureDictionaryClientHandler.getDictionary(key);
+ }
+
+ /**
+ * shutdown dictionary client
+ */
+ @Override public void shutDown() {
+ workerGroup.shutdownGracefully();
+ try {
+ workerGroup.terminationFuture().sync();
+ } catch (InterruptedException e) {
+ LOGGER.error(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClientHandler.java
new file mode 100644
index 0000000..3a76d84
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/NonSecureDictionaryClientHandler.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.dictionary.client;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+
+/**
+ * Client handler to get data.
+ */
+public class NonSecureDictionaryClientHandler extends ChannelInboundHandlerAdapter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(NonSecureDictionaryClientHandler.class.getName());
+
+ private final BlockingQueue<DictionaryMessage> responseMsgQueue = new LinkedBlockingQueue<>();
+
+ private ChannelHandlerContext ctx;
+
+ private DictionaryChannelFutureListener channelFutureListener;
+
+ @Override
+ public void channelActive(ChannelHandlerContext ctx) throws Exception {
+ this.ctx = ctx;
+ channelFutureListener = new DictionaryChannelFutureListener(ctx);
+ LOGGER.audit("Connected client " + ctx);
+ super.channelActive(ctx);
+ }
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ try {
+ ByteBuf data = (ByteBuf) msg;
+ DictionaryMessage key = new DictionaryMessage();
+ key.readSkipLength(data);
+ data.release();
+ responseMsgQueue.add(key);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ LOGGER.error(cause, "exceptionCaught");
+ ctx.close();
+ }
+
+ /**
+ * client send request to server
+ *
+ * @param key DictionaryMessage
+ * @return DictionaryMessage
+ */
+ public DictionaryMessage getDictionary(DictionaryMessage key) {
+ DictionaryMessage dictionaryMessage;
+ try {
+ ByteBuf buffer = ctx.alloc().buffer();
+ key.writeData(buffer);
+ ctx.writeAndFlush(buffer).addListener(channelFutureListener);
+ } catch (Exception e) {
+ LOGGER.error(e, "Error while send request to server ");
+ ctx.close();
+ }
+ try {
+ dictionaryMessage = responseMsgQueue.poll(100, TimeUnit.SECONDS);
+ if (dictionaryMessage == null) {
+ StringBuilder message = new StringBuilder();
+ message.append("DictionaryMessage { ColumnName: ")
+ .append(key.getColumnName())
+ .append(", DictionaryValue: ")
+ .append(key.getDictionaryValue())
+ .append(", type: ")
+ .append(key.getType());
+ throw new RuntimeException("Request timed out for key : " + message);
+ }
+ return dictionaryMessage;
+ } catch (Exception e) {
+ LOGGER.error(e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static class DictionaryChannelFutureListener implements ChannelFutureListener {
+
+ private ChannelHandlerContext ctx;
+
+ DictionaryChannelFutureListener(ChannelHandlerContext ctx) {
+ this.ctx = ctx;
+ }
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (!future.isSuccess()) {
+ LOGGER.error(future.cause(), "Error while sending request to Dictionary Server");
+ ctx.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index d59e9f8..3b3f21a 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -52,7 +52,32 @@ public class DictionaryMessage {
*/
private DictionaryMessageType type;
- public void readData(ByteBuf byteBuf) {
+ public void readSkipLength(ByteBuf byteBuf) {
+
+ byte[] tableBytes = new byte[byteBuf.readInt()];
+ byteBuf.readBytes(tableBytes);
+ tableUniqueId = new String(tableBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+
+ byte[] colBytes = new byte[byteBuf.readInt()];
+ byteBuf.readBytes(colBytes);
+ columnName = new String(colBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+
+ byte typeByte = byteBuf.readByte();
+ type = getKeyType(typeByte);
+
+ byte dataType = byteBuf.readByte();
+ if (dataType == 0) {
+ dictionaryValue = byteBuf.readInt();
+ } else {
+ byte[] dataBytes = new byte[byteBuf.readInt()];
+ byteBuf.readBytes(dataBytes);
+ data = new String(dataBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
+ }
+
+ public void readFullLength(ByteBuf byteBuf) {
+
+ byteBuf.readShort();
byte[] tableIdBytes = new byte[byteBuf.readInt()];
byteBuf.readBytes(tableIdBytes);
tableUniqueId =
@@ -107,6 +132,7 @@ public class DictionaryMessage {
byteBuf.setShort(startIndex, endIndex - startIndex - 2);
}
+
private DictionaryMessageType getKeyType(byte type) {
switch (type) {
case 2:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index d8a7c9e..a23cc9b 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -14,146 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.carbondata.core.dictionary.server;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
-import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-
-/**
- * Dictionary Server to generate dictionary keys.
- */
-public class DictionaryServer {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryServer.class.getName());
- private DictionaryServerHandler dictionaryServerHandler;
+public interface DictionaryServer {
- private EventLoopGroup boss;
- private EventLoopGroup worker;
- private int port;
- private static Object lock = new Object();
- private static volatile DictionaryServer INSTANCE = null;
+ public void startServer();
- private DictionaryServer(int port) {
- startServer(port);
- }
+ public void bindToPort();
- public static DictionaryServer getInstance(int port, CarbonTable carbonTable) throws Exception {
- if (INSTANCE == null) {
- synchronized (lock) {
- if (INSTANCE == null) {
- INSTANCE = new DictionaryServer(port);
- }
- }
- }
- INSTANCE.initializeDictionaryGenerator(carbonTable);
- return INSTANCE;
- }
+ public void shutdown()throws Exception;
- /**
- * start dictionary server
- *
- * @param port
- */
- private void startServer(int port) {
- dictionaryServerHandler = new DictionaryServerHandler();
- String workerThreads = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.DICTIONARY_WORKER_THREADS,
- CarbonCommonConstants.DICTIONARY_WORKER_THREADS_DEFAULT);
- boss = new NioEventLoopGroup(1);
- worker = new NioEventLoopGroup(Integer.parseInt(workerThreads));
- // Configure the server.
- bindToPort(port);
- }
+ public String getHost();
- /**
- * Binds dictionary server to an available port.
- *
- * @param port
- */
- private void bindToPort(int port) {
- long start = System.currentTimeMillis();
- // Configure the server.
- int i = 0;
- while (i < 10) {
- int newPort = port + i;
- try {
- ServerBootstrap bootstrap = new ServerBootstrap();
- bootstrap.group(boss, worker);
- bootstrap.channel(NioServerSocketChannel.class);
- bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
- @Override public void initChannel(SocketChannel ch) throws Exception {
- ChannelPipeline pipeline = ch.pipeline();
- pipeline
- .addLast("LengthDecoder",
- new LengthFieldBasedFrameDecoder(1048576, 0,
- 2, 0, 2));
- pipeline.addLast("DictionaryServerHandler", dictionaryServerHandler);
- }
- });
- bootstrap.childOption(ChannelOption.SO_KEEPALIVE, true);
- bootstrap.bind(newPort).sync();
- LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
- + " Listening on port " + newPort);
- this.port = newPort;
- break;
- } catch (Exception e) {
- LOGGER.error(e, "Dictionary Server Failed to bind to port:");
- if (i == 9) {
- throw new RuntimeException("Dictionary Server Could not bind to any port");
- }
- }
- i++;
- }
- }
+ public int getPort();
- /**
- *
- * @return Port on which the DictionaryServer has started.
- */
- public int getPort() {
- return port;
- }
+ public String getSecretKey();
- /**
- * shutdown dictionary server
- *
- * @throws Exception
- */
- public void shutdown() throws Exception {
- LOGGER.info("Shutting down dictionary server");
- worker.shutdownGracefully();
- boss.shutdownGracefully();
- }
+ public boolean isEncryptSecureServer();
- public void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception {
- dictionaryServerHandler.initializeTable(carbonTable);
- }
+ public void writeTableDictionary(String uniqueTableName) throws Exception;
- /**
- * Write Dictionary for one table.
- * @throws Exception
- */
- public void writeTableDictionary(String tableId) throws Exception {
- DictionaryMessage key = new DictionaryMessage();
- key.setTableUniqueId(tableId);
- key.setType(DictionaryMessageType.WRITE_TABLE_DICTIONARY);
- dictionaryServerHandler.processMessage(key);
- }
-}
\ No newline at end of file
+ public void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
deleted file mode 100644
index cf8581b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.dictionary.server;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.dictionary.generator.ServerDictionaryGenerator;
-import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInboundHandlerAdapter;
-
-/**
- * Handler for Dictionary server.
- */
-@ChannelHandler.Sharable public class DictionaryServerHandler extends ChannelInboundHandlerAdapter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryServerHandler.class.getName());
-
- /**
- * dictionary generator
- */
- private ServerDictionaryGenerator generatorForServer = new ServerDictionaryGenerator();
-
- /**
- * channel registered
- *
- * @param ctx
- * @throws Exception
- */
- public void channelActive(ChannelHandlerContext ctx) throws Exception {
- LOGGER.audit("Connected " + ctx);
- super.channelActive(ctx);
- }
-
- @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
- try {
- ByteBuf data = (ByteBuf) msg;
- DictionaryMessage key = new DictionaryMessage();
- key.readData(data);
- data.release();
- int outPut = processMessage(key);
- key.setDictionaryValue(outPut);
- // Send back the response
- ByteBuf buffer = ctx.alloc().buffer();
- key.writeData(buffer);
- ctx.writeAndFlush(buffer);
- } catch (Exception e) {
- LOGGER.error(e);
- throw e;
- }
- }
-
- /**
- * handle exceptions
- *
- * @param ctx
- * @param cause
- */
- @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- LOGGER.error(cause, "exceptionCaught");
- ctx.close();
- }
-
- /**
- * process message by message type
- *
- * @param key
- * @return
- * @throws Exception
- */
- public int processMessage(DictionaryMessage key) throws Exception {
- switch (key.getType()) {
- case DICT_GENERATION:
- return generatorForServer.generateKey(key);
- case SIZE:
- return generatorForServer.size(key);
- case WRITE_TABLE_DICTIONARY:
- generatorForServer
- .writeTableDictionaryData(key.getTableUniqueId());
- return 0;
- default:
- return -1;
- }
- }
-
- void initializeTable(CarbonTable carbonTable) {
- generatorForServer.initializeGeneratorForTable(carbonTable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
new file mode 100644
index 0000000..c7411d6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.dictionary.server;
+
+import java.net.InetSocketAddress;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType;
+import org.apache.carbondata.core.dictionary.service.AbstractDictionaryServer;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
+
+/**
+ * Dictionary Server to generate dictionary keys.
+ */
+public class NonSecureDictionaryServer extends AbstractDictionaryServer
+ implements DictionaryServer {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(NonSecureDictionaryServer.class.getName());
+
+ private NonSecureDictionaryServerHandler nonSecureDictionaryServerHandler;
+
+ private EventLoopGroup boss;
+ private EventLoopGroup worker;
+ private int port;
+ private String host;
+ private static Object lock = new Object();
+ private static NonSecureDictionaryServer INSTANCE = null;
+
+ private NonSecureDictionaryServer(int port) {
+ this.port = port;
+ startServer();
+ }
+
+ public static synchronized DictionaryServer getInstance(int port, CarbonTable carbonTable)
+ throws Exception {
+ if (INSTANCE == null) {
+ INSTANCE = new NonSecureDictionaryServer(port);
+ }
+ INSTANCE.initializeDictionaryGenerator(carbonTable);
+ return INSTANCE;
+ }
+
+ /**
+ * start dictionary server
+ *
+ */
+ @Override public void startServer() {
+ LOGGER.info("Starting Dictionary Server in Non Secure Mode");
+ nonSecureDictionaryServerHandler = new NonSecureDictionaryServerHandler();
+ String workerThreads = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.DICTIONARY_WORKER_THREADS,
+ CarbonCommonConstants.DICTIONARY_WORKER_THREADS_DEFAULT);
+ boss = new NioEventLoopGroup(1);
+ worker = new NioEventLoopGroup(Integer.parseInt(workerThreads));
+ // Configure the server.
+ bindToPort();
+ }
+
+ /**
+ * Binds dictionary server to an available port.
+ *
+ */
+ @Override public void bindToPort() {
+ long start = System.currentTimeMillis();
+ // Configure the server.
+ int i = 0;
+ while (i < 10) {
+ int newPort = port + i;
+ try {
+ ServerBootstrap bootstrap = new ServerBootstrap();
+ bootstrap.group(boss, worker);
+ bootstrap.channel(NioServerSocketChannel.class);
+ bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
+ @Override public void initChannel(SocketChannel ch) throws Exception {
+ ChannelPipeline pipeline = ch.pipeline();
+ pipeline
+ .addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
+ pipeline.addLast("NonSecureDictionaryServerHandler", nonSecureDictionaryServerHandler);
+ }
+ });
+ bootstrap.childOption(ChannelOption.SO_KEEPALIVE, true);
+ String hostToBind = findLocalIpAddress(LOGGER);
+ InetSocketAddress address = hostToBind == null ?
+ new InetSocketAddress(newPort) :
+ new InetSocketAddress(hostToBind, newPort);
+ bootstrap.bind(address).sync();
+ LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
+ + " Listening on port " + newPort);
+ this.port = newPort;
+ this.host = hostToBind;
+ break;
+ } catch (Exception e) {
+ LOGGER.error(e, "Dictionary Server Failed to bind to port:");
+ if (i == 9) {
+ throw new RuntimeException("Dictionary Server Could not bind to any port");
+ }
+ }
+ i++;
+ }
+ }
+
+ /**
+ * @return Port on which the NonSecureDictionaryServer has started.
+ */
+ @Override public int getPort() {
+ return port;
+ }
+
+ @Override public String getSecretKey() {
+ return null;
+ }
+
+ @Override public boolean isEncryptSecureServer() {
+ return false;
+ }
+
+ @Override public String getHost() {
+ return host;
+ }
+
+ /**
+ * shutdown dictionary server
+ *
+ * @throws Exception
+ */
+ @Override public void shutdown() throws Exception {
+ LOGGER.info("Shutting down dictionary server");
+ worker.shutdownGracefully();
+ boss.shutdownGracefully();
+ }
+
+ /**
+ * Write Dictionary for one table.
+ *
+ * @throws Exception
+ */
+
+ @Override
+ public void writeTableDictionary(String uniqueTableName) throws Exception {
+ DictionaryMessage key = new DictionaryMessage();
+ key.setTableUniqueId(uniqueTableName);
+ key.setType(DictionaryMessageType.WRITE_TABLE_DICTIONARY);
+ nonSecureDictionaryServerHandler.processMessage(key);
+ }
+
+ public void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception {
+ nonSecureDictionaryServerHandler.initializeTable(carbonTable);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
new file mode 100644
index 0000000..dc3d078
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.dictionary.server;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.dictionary.generator.ServerDictionaryGenerator;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+
+/**
+ * Handler for Dictionary server.
+ */
+@ChannelHandler.Sharable public class NonSecureDictionaryServerHandler
+ extends ChannelInboundHandlerAdapter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(NonSecureDictionaryServerHandler.class.getName());
+
+ /**
+ * dictionary generator
+ */
+ private ServerDictionaryGenerator generatorForServer = new ServerDictionaryGenerator();
+
+ /**
+ * channel registered
+ *
+ * @param ctx
+ * @throws Exception
+ */
+ public void channelActive(ChannelHandlerContext ctx) throws Exception {
+ LOGGER.audit("Connected " + ctx);
+ super.channelActive(ctx);
+ }
+
+ @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ try {
+ ByteBuf data = (ByteBuf) msg;
+ DictionaryMessage key = new DictionaryMessage();
+ key.readSkipLength(data);
+ data.release();
+ int outPut = processMessage(key);
+ key.setDictionaryValue(outPut);
+ // Send back the response
+ ByteBuf buffer = ctx.alloc().buffer();
+ key.writeData(buffer);
+ ctx.writeAndFlush(buffer);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ throw e;
+ }
+ }
+
+ /**
+ * handle exceptions
+ *
+ * @param ctx
+ * @param cause
+ */
+ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ LOGGER.error(cause, "exceptionCaught");
+ ctx.close();
+ }
+
+ /**
+ * process message by message type
+ *
+ * @param key
+ * @return
+ * @throws Exception
+ */
+ public int processMessage(DictionaryMessage key) throws Exception {
+ switch (key.getType()) {
+ case DICT_GENERATION:
+ generatorForServer.initializeGeneratorForColumn(key);
+ return generatorForServer.generateKey(key);
+ case SIZE:
+ generatorForServer.initializeGeneratorForColumn(key);
+ return generatorForServer.size(key);
+ case WRITE_TABLE_DICTIONARY:
+ generatorForServer
+ .writeTableDictionaryData(key.getTableUniqueId());
+ return 0;
+ default:
+ return -1;
+ }
+ }
+
+ void initializeTable(CarbonTable carbonTable) {
+ generatorForServer.initializeGeneratorForTable(carbonTable);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/service/AbstractDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/service/AbstractDictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/service/AbstractDictionaryServer.java
new file mode 100644
index 0000000..754f253
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/service/AbstractDictionaryServer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.dictionary.service;
+
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+
+import org.apache.commons.lang3.SystemUtils;
+
+public abstract class AbstractDictionaryServer {
+
+ public String findLocalIpAddress(LogService LOGGER) {
+ try {
+ String defaultIpOverride = System.getenv("SPARK_LOCAL_IP");
+ if (defaultIpOverride != null) {
+ return defaultIpOverride;
+ } else {
+ InetAddress address = InetAddress.getLocalHost();
+ if (address.isLoopbackAddress()) {
+ // Address resolves to something like 127.0.1.1, which happens on Debian; try to find
+ // a better address using the local network interfaces
+ // getNetworkInterfaces returns ifs in reverse order compared to ifconfig output order
+ // on unix-like system. On windows, it returns in index order.
+ // It's more proper to pick ip address following system output order.
+ Enumeration<NetworkInterface> activeNetworkIFs = NetworkInterface.getNetworkInterfaces();
+ List<NetworkInterface> reOrderedNetworkIFs = new ArrayList<NetworkInterface>();
+ while (activeNetworkIFs.hasMoreElements()) {
+ reOrderedNetworkIFs.add(activeNetworkIFs.nextElement());
+ }
+
+ if (!SystemUtils.IS_OS_WINDOWS) {
+ Collections.reverse(reOrderedNetworkIFs);
+ }
+
+ for (NetworkInterface ni : reOrderedNetworkIFs) {
+ Enumeration<InetAddress> inetAddresses = ni.getInetAddresses();
+ while (inetAddresses.hasMoreElements()) {
+ InetAddress addr = inetAddresses.nextElement();
+ if (!addr.isLinkLocalAddress() && !addr.isLoopbackAddress()
+ && addr instanceof Inet4Address) {
+ // We've found an address that looks reasonable!
+ LOGGER.warn("Your hostname, " + InetAddress.getLocalHost().getHostName()
+ + " resolves to a loopback address: " + address.getHostAddress() + "; using "
+ + addr.getHostAddress() + " instead (on interface " + ni.getName() + ")");
+ LOGGER.warn("Set SPARK_LOCAL_IP if you need to bind to another address");
+ return addr.getHostAddress();
+ }
+ }
+ LOGGER.warn("Your hostname, " + InetAddress.getLocalHost().getHostName()
+ + " resolves to a loopback address: " + address.getHostAddress()
+ + ", but we couldn't find any external IP address!");
+ LOGGER.warn("Set SPARK_LOCAL_IP if you need to bind to another address");
+ }
+ }
+ return address.getHostAddress();
+ }
+ } catch (UnknownHostException e) {
+ LOGGER.error("do not get local host address:" + e.getMessage());
+ throw new RuntimeException(e.getMessage());
+ } catch (SocketException e) {
+ LOGGER.error("do not get net work interface:" + e.getMessage());
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryOnePassService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryOnePassService.java b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryOnePassService.java
new file mode 100644
index 0000000..1625f40
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryOnePassService.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.dictionary.service;
+
+public class DictionaryOnePassService {
+ private static DictionaryServiceProvider dictionaryServiceProvider = null;
+
+ public static void setDictionaryServiceProvider(DictionaryServiceProvider dictionaryServiceProv) {
+ dictionaryServiceProvider = dictionaryServiceProv;
+ }
+
+ public static synchronized DictionaryServiceProvider getDictionayProvider() {
+ return dictionaryServiceProvider;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
new file mode 100644
index 0000000..4dfa1e9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.dictionary.service;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.dictionary.client.DictionaryClient;
+
+public interface DictionaryServiceProvider extends Serializable {
+
+ public DictionaryClient getDictionaryClient();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/main/java/org/apache/carbondata/core/dictionary/service/NonSecureDictionaryServiceProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/service/NonSecureDictionaryServiceProvider.java b/core/src/main/java/org/apache/carbondata/core/dictionary/service/NonSecureDictionaryServiceProvider.java
new file mode 100644
index 0000000..70354cb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/service/NonSecureDictionaryServiceProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.dictionary.service;
+
+import org.apache.carbondata.core.dictionary.client.DictionaryClient;
+import org.apache.carbondata.core.dictionary.client.NonSecureDictionaryClient;
+
+public class NonSecureDictionaryServiceProvider implements DictionaryServiceProvider {
+ private int port = 0;
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public NonSecureDictionaryServiceProvider(int port) {
+ this.port = port;
+ }
+
+ // @Override public DictionaryServer getDictionaryServer() {
+ // return NonSecureDictionaryServer.getInstance(port);
+ // }
+
+ @Override public DictionaryClient getDictionaryClient() {
+ return new NonSecureDictionaryClient();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index 5318f3d..2fd69f8 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType;
import org.apache.carbondata.core.dictionary.server.DictionaryServer;
+import org.apache.carbondata.core.dictionary.server.NonSecureDictionaryServer;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -56,6 +57,7 @@ public class DictionaryClientTest {
private static TableInfo tableInfo;
private static String storePath;
private static DictionaryServer server;
+ private static String host;
@BeforeClass public static void setUp() throws Exception {
// enable lru cache by setting cache size
@@ -96,12 +98,13 @@ public class DictionaryClientTest {
metadata.addCarbonTable(carbonTable);
// Start the server for testing the client
- server = DictionaryServer.getInstance(5678, carbonTable);
+ server = NonSecureDictionaryServer.getInstance(5678, carbonTable);
+ host = server.getHost();
}
@Test public void testClient() throws Exception {
- DictionaryClient client = new DictionaryClient();
- client.startClient("localhost", 5678);
+ NonSecureDictionaryClient client = new NonSecureDictionaryClient();
+ client.startClient(null, host, 5678, false);
Thread.sleep(1000);
// Create a dictionary key
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index a42b366..76afcbf 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -62,9 +62,9 @@ object CarbonSessionExample {
spark.sql(
s"""
- | SELECT *
- | FROM carbon_table
- | WHERE stringfield = 'spark' AND decimalField > 40
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringfield = 'spark' AND decimalField > 40
""".stripMargin).show()
spark.sql(
@@ -90,21 +90,21 @@ object CarbonSessionExample {
spark.sql(
s"""
- | SELECT t1.*, t2.*
- | FROM carbon_table t1, carbon_table t2
- | WHERE t1.stringField = t2.stringField
+ | SELECT t1.*, t2.*
+ | FROM carbon_table t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
""".stripMargin).show()
spark.sql(
s"""
- | WITH t1 AS (
- | SELECT * FROM carbon_table
- | UNION ALL
- | SELECT * FROM carbon_table
- | )
- | SELECT t1.*, t2.*
- | FROM t1, carbon_table t2
- | WHERE t1.stringField = t2.stringField
+ | WITH t1 AS (
+ | SELECT * FROM carbon_table
+ | UNION ALL
+ | SELECT * FROM carbon_table
+ | )
+ | SELECT t1.*, t2.*
+ | FROM t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
""".stripMargin).show()
spark.sql(
@@ -120,4 +120,4 @@ object CarbonSessionExample {
spark.stop()
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClient.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClient.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClient.java
new file mode 100644
index 0000000..1c8e2d2
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClient.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.dictionary.client;
+
+import java.nio.charset.Charset;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.dictionary.client.DictionaryClient;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+
+import com.google.common.collect.Lists;
+import io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.spark.SecurityManager;
+import org.apache.spark.SparkConf;
+import org.apache.spark.network.TransportContext;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.client.TransportClientBootstrap;
+import org.apache.spark.network.client.TransportClientFactory;
+import org.apache.spark.network.netty.SparkTransportConf;
+import org.apache.spark.network.sasl.SaslClientBootstrap;
+import org.apache.spark.network.util.TransportConf;
+
+/**
+ * Dictionary client to connect to Dictionary server and generate dictionary values
+ */
+public class SecureDictionaryClient implements DictionaryClient {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SecureDictionaryClient.class.getName());
+
+ private SecureDictionaryClientHandler dictionaryClientHandler =
+ new SecureDictionaryClientHandler();
+
+ private NioEventLoopGroup workerGroup;
+ private TransportClient client;
+ private TransportClientFactory clientFactory;
+
+ /**
+ * start dictionary client
+ *
+ * @param address
+ * @param port
+ */
+ @Override public void startClient(String secretKey, String address, int port,
+ boolean encryptSecureServer) {
+ LOGGER.audit("Starting client on " + address + " " + port);
+ long start = System.currentTimeMillis();
+
+ SecurityManager securityMgr;
+ SparkConf conf = new SparkConf().setAppName("Carbon Dictionary Client");
+
+ conf.set("spark.authenticate", "true");
+
+ if (null != secretKey) {
+ conf.set("spark.authenticate.secret", secretKey);
+ }
+
+ if (encryptSecureServer) {
+ conf.set("spark.authenticate.enableSaslEncryption", "true");
+ }
+
+ TransportConf transportConf =
+ SparkTransportConf.fromSparkConf(conf, "Carbon Dictionary Client", 0);
+ if (null != secretKey) {
+ securityMgr = new SecurityManager(conf, scala.Option.apply(secretKey.getBytes(Charset.forName(
+ CarbonCommonConstants.DEFAULT_CHARSET))));
+ } else {
+ securityMgr = new SecurityManager(conf, null);
+ }
+
+ TransportContext context = new TransportContext(transportConf, dictionaryClientHandler);
+ clientFactory = context.createClientFactory(Lists.<TransportClientBootstrap>newArrayList(
+ new SaslClientBootstrap(transportConf, "Carbon Dictionary Client", securityMgr)));
+
+ try {
+ client = clientFactory.createClient(address, port);
+ } catch (Exception e) {
+ LOGGER.error(e, "Dictionary Client Failed to bind to port:");
+ }
+ LOGGER.info(
+ "Dictionary client Started, Total time spent : " + (System.currentTimeMillis() - start));
+ }
+
+ /**
+ * for client request
+ *
+ * @param key
+ * @return
+ */
+ @Override public DictionaryMessage getDictionary(DictionaryMessage key) {
+ return dictionaryClientHandler.getDictionary(key, this.client);
+ }
+
+ /**
+ * shutdown dictionary client
+ */
+ @Override public void shutDown() {
+ clientFactory.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClientHandler.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClientHandler.java
new file mode 100644
index 0000000..cdf2553
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/client/SecureDictionaryClientHandler.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.dictionary.client;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.server.OneForOneStreamManager;
+import org.apache.spark.network.server.RpcHandler;
+import org.apache.spark.network.server.StreamManager;
+
+/**
+ * Client handler to get data.
+ */
+public class SecureDictionaryClientHandler extends RpcHandler {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SecureDictionaryClientHandler.class.getName());
+
+ private final BlockingQueue<DictionaryMessage> responseMsgQueue = new LinkedBlockingQueue<>();
+
+ /**
+ * client send request to server
+ *
+ * @param key DictionaryMessage
+ * @return DictionaryMessage
+ */
+ public DictionaryMessage getDictionary(DictionaryMessage key, TransportClient client) {
+ DictionaryMessage dictionaryMessage;
+ ByteBuffer resp = null;
+ try {
+
+ ByteBuf buffer = ByteBufAllocator.DEFAULT.heapBuffer();
+ key.writeData(buffer);
+ resp = client.sendRpcSync(buffer.nioBuffer(), 100000);
+ } catch (Exception e) {
+ LOGGER.error(e, "Error while send request to server ");
+ }
+ try {
+ if (resp == null) {
+ StringBuilder message = new StringBuilder();
+ message.append("DictionaryMessage { ColumnName: ").append(key.getColumnName())
+ .append(", DictionaryValue: ").append(key.getDictionaryValue()).append(", type: ")
+ .append(key.getType()).append(" }");
+ throw new RuntimeException("Request timed out for key : " + message);
+ }
+ DictionaryMessage newKey = new DictionaryMessage();
+ ByteBuf data = Unpooled.wrappedBuffer(resp);
+ newKey.readFullLength(data);
+ data.release();
+ return newKey;
+ } catch (Exception e) {
+ LOGGER.error(e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public void receive(TransportClient transportClient, ByteBuffer byteBuffer,
+ RpcResponseCallback rpcResponseCallback) {
+ try {
+ ByteBuf data = Unpooled.wrappedBuffer(byteBuffer);
+ DictionaryMessage key = new DictionaryMessage();
+ key.readFullLength(data);
+ data.release();
+ if (responseMsgQueue.offer(key)) {
+ LOGGER.info("key: " + key + " added to queue");
+ } else {
+ LOGGER.error("Failed to add key: " + key + " to queue");
+ }
+ } catch (Exception e) {
+ LOGGER.error(e);
+ throw e;
+ }
+ }
+
+ @Override public StreamManager getStreamManager() {
+ return new OneForOneStreamManager();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/provider/SecureDictionaryServiceProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/provider/SecureDictionaryServiceProvider.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/provider/SecureDictionaryServiceProvider.java
new file mode 100644
index 0000000..3db344b
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/provider/SecureDictionaryServiceProvider.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.dictionary.provider;
+
+import org.apache.carbondata.core.dictionary.client.DictionaryClient;
+import org.apache.carbondata.core.dictionary.service.DictionaryServiceProvider;
+import org.apache.carbondata.spark.dictionary.client.SecureDictionaryClient;
+
+public class SecureDictionaryServiceProvider implements DictionaryServiceProvider {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override public DictionaryClient getDictionaryClient() {
+ return new SecureDictionaryClient();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServer.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServer.java
new file mode 100644
index 0000000..6dd6581
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServer.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.dictionary.server;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType;
+import org.apache.carbondata.core.dictionary.server.DictionaryServer;
+import org.apache.carbondata.core.dictionary.service.AbstractDictionaryServer;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import com.google.common.collect.Lists;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.spark.SecurityManager;
+import org.apache.spark.SparkConf;
+import org.apache.spark.network.TransportContext;
+import org.apache.spark.network.netty.SparkTransportConf;
+import org.apache.spark.network.sasl.SaslServerBootstrap;
+import org.apache.spark.network.server.TransportServer;
+import org.apache.spark.network.server.TransportServerBootstrap;
+import org.apache.spark.network.util.TransportConf;
+import scala.Some;
+
+/**
+ * Dictionary Server to generate dictionary keys.
+ */
+public class SecureDictionaryServer extends AbstractDictionaryServer implements DictionaryServer {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SecureDictionaryServer.class.getName());
+
+ private SecureDictionaryServerHandler secureDictionaryServerHandler;
+
+ private EventLoopGroup boss;
+ private EventLoopGroup worker;
+ private int port;
+ private String host;
+ private SparkConf conf;
+ private String secretKey = null;
+ private boolean encryptSecureServer;
+ private static Object lock = new Object();
+ private static SecureDictionaryServer INSTANCE = null;
+
+ private SecureDictionaryServer(SparkConf conf, String host, int port) {
+ this.conf = conf;
+ this.host = host;
+ this.port = port;
+ startServer();
+ }
+
+ public static synchronized DictionaryServer getInstance(SparkConf conf, String host, int port,
+ CarbonTable carbonTable) throws Exception {
+ if (INSTANCE == null) {
+ INSTANCE = new SecureDictionaryServer(conf, host, port);
+ }
+ INSTANCE.initializeDictionaryGenerator(carbonTable);
+ return INSTANCE;
+ }
+
+ /**
+ * start dictionary server
+ *
+ */
+ @Override
+ public void startServer() {
+ LOGGER.info("Starting Dictionary Server in Secure Mode");
+ secureDictionaryServerHandler = new SecureDictionaryServerHandler();
+ String workerThreads = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.DICTIONARY_WORKER_THREADS,
+ CarbonCommonConstants.DICTIONARY_WORKER_THREADS_DEFAULT);
+ boss = new NioEventLoopGroup(1);
+ worker = new NioEventLoopGroup(Integer.parseInt(workerThreads));
+ // Configure the server.
+ bindToPort();
+ }
+
+ /**
+ * Binds dictionary server to an available port.
+ *
+ */
+ @Override
+ public void bindToPort() {
+ long start = System.currentTimeMillis();
+ // Configure the server.
+ int i = 0;
+ while (i < 10) {
+ int newPort = port + i;
+ try {
+ SecurityManager securityManager;
+ SparkConf conf = this.conf.clone();
+ conf.setAppName("Carbon Dictionary Server");
+
+ // As spark.network.sasl.serverAlwaysEncrypt is not exposed parameter
+ // set it explicitly so that Dictionary Server and Client communication
+ // occurs encrypted. The below parameter can be removed once spark Documents it.
+ // conf.set("spark.network.sasl.serverAlwaysEncrypt", "true");
+ conf.set("spark.authenticate.enableSaslEncryption", "true");
+
+ if (conf.get("spark.authenticate.enableSaslEncryption", "false").equalsIgnoreCase("true")) {
+ setEncryptSecureServer(true);
+ } else {
+ setEncryptSecureServer(false);
+ }
+
+ TransportConf transportConf =
+ SparkTransportConf.fromSparkConf(conf, "Carbon Dictionary Server", 0);
+ securityManager = new SecurityManager(conf, Some.<byte[]>empty());
+ secretKey = securityManager.getSecretKey();
+ TransportContext context =
+ new TransportContext(transportConf, secureDictionaryServerHandler);
+ TransportServerBootstrap bootstrap =
+ new SaslServerBootstrap(transportConf, securityManager);
+ String host = findLocalIpAddress(LOGGER);
+ TransportServer transportServer = context
+ .createServer(host, port, Lists.<TransportServerBootstrap>newArrayList(bootstrap));
+ LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
+ + " Listening on port " + newPort);
+ this.port = newPort;
+ this.host = host;
+ break;
+ } catch (Exception e) {
+ LOGGER.error(e, "Dictionary Server Failed to bind to port:");
+ if (i == 9) {
+ throw new RuntimeException("Dictionary Server Could not bind to any port");
+ }
+ }
+ i++;
+ }
+ }
+
+ private void setEncryptSecureServer(boolean encryptSecureServer) {
+ this.encryptSecureServer = encryptSecureServer;
+ }
+
+ /**
+ * @return Port on which the SecureDictionaryServer has started.
+ */
+ @Override
+ public int getPort() {
+ return port;
+ }
+
+ /**
+ * @return IP address on which the SecureDictionaryServer has Started.
+ */
+ @Override
+ public String getHost() {
+ return host;
+ }
+
+ /**
+ * @return Secret Key of Dictionary Server.
+ */
+ @Override
+ public String getSecretKey() {
+ return secretKey;
+ }
+
+ @Override public boolean isEncryptSecureServer() {
+ return encryptSecureServer;
+ }
+
+ /**
+ * shutdown dictionary server
+ *
+ * @throws Exception
+ */
+ @Override
+ public void shutdown() throws Exception {
+ LOGGER.info("Shutting down dictionary server");
+ worker.shutdownGracefully();
+ boss.shutdownGracefully();
+ }
+
+ public void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception {
+ secureDictionaryServerHandler.initializeTable(carbonTable);
+ }
+
+ /**
+ * Write Dictionary for one table.
+ *
+ * @throws Exception
+ */
+
+ @Override
+ public void writeTableDictionary(String uniqueTableName) throws Exception {
+ DictionaryMessage key = new DictionaryMessage();
+ key.setTableUniqueId(uniqueTableName);
+ key.setType(DictionaryMessageType.WRITE_TABLE_DICTIONARY);
+ secureDictionaryServerHandler.processMessage(key);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServerHandler.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServerHandler.java
new file mode 100644
index 0000000..aaa4cf0
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/dictionary/server/SecureDictionaryServerHandler.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.dictionary.server;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.dictionary.generator.ServerDictionaryGenerator;
+import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandler;
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.server.OneForOneStreamManager;
+import org.apache.spark.network.server.RpcHandler;
+import org.apache.spark.network.server.StreamManager;
+
+/**
+ * Handler for Dictionary server.
+ */
+@ChannelHandler.Sharable public class SecureDictionaryServerHandler extends RpcHandler {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SecureDictionaryServerHandler.class.getName());
+
+ /**
+ * dictionary generator
+ */
+ private ServerDictionaryGenerator generatorForServer = new ServerDictionaryGenerator();
+
+ /**
+ * process message by message type
+ *
+ * @param key
+ * @return
+ * @throws Exception
+ */
+ public int processMessage(DictionaryMessage key) throws Exception {
+ switch (key.getType()) {
+ case DICT_GENERATION:
+ generatorForServer.initializeGeneratorForColumn(key);
+ return generatorForServer.generateKey(key);
+ case SIZE:
+ generatorForServer.initializeGeneratorForColumn(key);
+ return generatorForServer.size(key);
+ case WRITE_TABLE_DICTIONARY:
+ generatorForServer.writeTableDictionaryData(key.getTableUniqueId());
+ return 0;
+ default:
+ return -1;
+ }
+ }
+
+ @Override public void receive(TransportClient transportClient, ByteBuffer byteBuffer,
+ RpcResponseCallback rpcResponseCallback) {
+ try {
+ ByteBuf data = Unpooled.wrappedBuffer(byteBuffer);
+ DictionaryMessage key = new DictionaryMessage();
+ key.readFullLength(data);
+ data.release();
+ int outPut = processMessage(key);
+ key.setDictionaryValue(outPut);
+ // Send back the response
+ ByteBuf buff = ByteBufAllocator.DEFAULT.buffer();
+ key.writeData(buff);
+ rpcResponseCallback.onSuccess(buff.nioBuffer());
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+ }
+
+ @Override public StreamManager getStreamManager() {
+ return new OneForOneStreamManager();
+ }
+
+ public void initializeTable(CarbonTable carbonTable) {
+ generatorForServer.initializeGeneratorForTable(carbonTable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4daf0634/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
new file mode 100644
index 0000000..e69de29
[41/50] [abbrv] carbondata git commit: [CARBONDATA-1884] SDV test
cases for CTAS support to carbondata
Posted by ja...@apache.org.
[CARBONDATA-1884] SDV test cases for CTAS support to carbondata
SDV test cases for CTAS support to carbondata.
This closes #1696
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4b0eee3b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4b0eee3b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4b0eee3b
Branch: refs/heads/fgdatamap
Commit: 4b0eee3b4cdad69fd11985a650bcb62ff0e280f0
Parents: 6add19a
Author: Pawan Malwal <pm...@gmail.com>
Authored: Wed Dec 20 15:30:58 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Fri Dec 22 16:48:05 2017 +0530
----------------------------------------------------------------------
.../generated/CreateTableAsSelectTestCase.scala | 156 +++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 2 +
2 files changed, 158 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b0eee3b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableAsSelectTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableAsSelectTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableAsSelectTestCase.scala
new file mode 100644
index 0000000..aa8c404
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableAsSelectTestCase.scala
@@ -0,0 +1,156 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.common.util._
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for for create table as select command
+ */
+
+class CreateTableAsSelectTestCase extends QueryTest with BeforeAndAfterAll {
+
+
+ //Check create table as select with select from same table name when table exists
+ test("CreateTableAsSelect_001_01", Include) {
+ sql("drop table if exists ctas_same_table_name").collect
+ sql("CREATE TABLE ctas_same_table_name(key INT, value STRING) STORED by 'carbondata'").collect
+ intercept[Exception] {
+ sql("create table ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name")
+ }
+ }
+
+ //Check create table as select with select from same table name when table does not exists
+ test("CreateTableAsSelect_001_02", Include) {
+ sql("drop table if exists ctas_same_table_name").collect
+ intercept[Exception] {
+ sql("create table ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name")
+ }
+ }
+
+ //Check create table as select with select from same table name with if not exists clause
+ test("CreateTableAsSelect_001_03", Include) {
+ sql("drop table if exists ctas_same_table_name").collect
+ sql("CREATE TABLE ctas_same_table_name(key INT, value STRING) STORED by 'carbondata'").collect
+ sql("create table if not exists ctas_same_table_name stored by 'carbondata' as select * from ctas_same_table_name").collect
+ assert(true)
+ }
+
+ //Check create table as select with select from another carbon table
+ test("CreateTableAsSelect_001_04", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_carbon").collect
+ sql("create table ctas_select_carbon stored by 'carbondata' as select * from carbon_ctas_test").collect
+ checkAnswer(sql("select * from ctas_select_carbon"), sql("select * from carbon_ctas_test"))
+ }
+
+ //Check create table as select with select from another parquet table
+ test("CreateTableAsSelect_001_05", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_parquet").collect
+ sql("create table ctas_select_parquet stored by 'carbondata' as select * from parquet_ctas_test").collect
+ checkAnswer(sql("select * from ctas_select_parquet"), sql("select * from parquet_ctas_test"))
+ }
+
+ //Check test create table as select with select from another hive/orc table
+ test("CreateTableAsSelect_001_06", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_orc").collect
+ sql("create table ctas_select_orc stored by 'carbondata' as select * from orc_ctas_test").collect
+ checkAnswer(sql("select * from ctas_select_orc"), sql("select * from orc_ctas_test"))
+ }
+
+ //Check create table as select with where clause in select from carbon table that returns data
+ test("CreateTableAsSelect_001_07", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon").collect
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=100").collect
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test where key=100"))
+ }
+
+ //Check create table as select with where clause in select from carbon table that does not return data
+ test("CreateTableAsSelect_001_08", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon").collect
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=300").collect
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test where key=300"))
+ }
+
+ //Check create table as select with where clause in select from carbon table and load again
+ test("CreateTableAsSelect_001_09", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_where_carbon").collect
+ sql("create table ctas_select_where_carbon stored by 'carbondata' as select * from carbon_ctas_test where key=100").collect
+ sql("insert into ctas_select_where_carbon select 200,'hive'").collect
+ checkAnswer(sql("select * from ctas_select_where_carbon"), sql("select * from carbon_ctas_test"))
+ }
+
+ //Check create table as select with where clause in select from parquet table
+ test("CreateTableAsSelect_001_10", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_where_parquet").collect
+ sql("create table ctas_select_where_parquet stored by 'carbondata' as select * from parquet_ctas_test where key=100").collect
+ checkAnswer(sql("select * from ctas_select_where_parquet"), sql("select * from parquet_ctas_test where key=100"))
+ }
+
+ //Check create table as select with where clause in select from hive/orc table
+ test("CreateTableAsSelect_001_11", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_where_orc").collect
+ sql("create table ctas_select_where_orc stored by 'carbondata' as select * from orc_ctas_test where key=100").collect
+ checkAnswer(sql("select * from ctas_select_where_orc"), sql("select * from orc_ctas_test where key=100"))
+ }
+
+ //Check create table as select with select directly having the data
+ test("CreateTableAsSelect_001_12", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_direct_data").collect
+ sql("create table ctas_select_direct_data stored by 'carbondata' as select 300,'carbondata'").collect
+ checkAnswer(sql("select * from ctas_select_direct_data"), Seq(Row(300,"carbondata")))
+ }
+
+ //Check create table as select with select from another carbon table with more data
+ test("CreateTableAsSelect_001_13", Include) {
+ sql("DROP TABLE IF EXISTS ctas_select_hugedata1").collect
+ sql("DROP TABLE IF EXISTS ctas_select_hugedata2").collect
+ sql(s"""CREATE TABLE ctas_select_hugedata1 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table ctas_select_hugedata1 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql("create table ctas_select_hugedata2 stored by 'carbondata' as select * from ctas_select_hugedata1").collect
+ checkAnswer(sql("select * from ctas_select_hugedata1"), sql("select * from ctas_select_hugedata2"))
+ sql("DROP TABLE IF EXISTS ctas_select_hugedata1").collect
+ sql("DROP TABLE IF EXISTS ctas_select_hugedata2").collect
+ }
+
+ override protected def beforeAll() {
+ // Dropping existing tables
+ sql("DROP TABLE IF EXISTS carbon_ctas_test")
+ sql("DROP TABLE IF EXISTS parquet_ctas_test")
+ sql("DROP TABLE IF EXISTS orc_ctas_test")
+
+ // create carbon table and insert data
+ sql("CREATE TABLE carbon_ctas_test(key INT, value STRING) STORED by 'carbondata'")
+ sql("insert into carbon_ctas_test select 100,'spark'")
+ sql("insert into carbon_ctas_test select 200,'hive'")
+
+ // create parquet table and insert data
+ sql("CREATE TABLE parquet_ctas_test(key INT, value STRING) STORED as parquet")
+ sql("insert into parquet_ctas_test select 100,'spark'")
+ sql("insert into parquet_ctas_test select 200,'hive'")
+
+ // create hive table and insert data
+ sql("CREATE TABLE orc_ctas_test(key INT, value STRING) STORED as ORC")
+ sql("insert into orc_ctas_test select 100,'spark'")
+ sql("insert into orc_ctas_test select 200,'hive'")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b0eee3b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index b968ae6..32f74fb 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -33,6 +33,7 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
new BatchSortLoad2TestCase ::
new BatchSortQueryTestCase ::
new ColumndictTestCase ::
+ new CreateTableAsSelectTestCase ::
new DataLoadingTestCase ::
new DataLoadingV3TestCase ::
new InvertedindexTestCase ::
@@ -119,6 +120,7 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
val suites = new AlterTableTestCase ::
new BatchSortLoad2TestCase ::
new BucketingTestCase ::
+ new CreateTableAsSelectTestCase ::
new InvertedindexTestCase ::
new OffheapQuery1TestCase ::
new OffheapQuery2TestCase ::
[43/50] [abbrv] carbondata git commit: [CARBONDATA-1930] Added
condition to refer to parent dictionary if filter is given in aggregate table
query
Posted by ja...@apache.org.
[CARBONDATA-1930] Added condition to refer to parent dictionary if filter is given in aggregate table query
When filter is applied to aggregate query then the query is not accessing the parent table dictionary instead it is trying to search for aggregate table dictionary files.Add check to access parent table dictionary files if aggregate column has parent columns.
This closes #1710
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1381c9f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1381c9f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1381c9f1
Branch: refs/heads/fgdatamap
Commit: 1381c9f19508f68f95953c48126e27ca239818d1
Parents: 406d754
Author: kunal642 <ku...@gmail.com>
Authored: Fri Dec 22 13:23:11 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Fri Dec 22 21:41:31 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/scan/filter/FilterUtil.java | 16 +++++++++++++++-
.../TestPreAggregateTableSelection.scala | 11 +++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index a8fa8ec..3268ca3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -61,6 +62,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -1241,15 +1243,27 @@ public final class FilterUtil {
AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier,
CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException {
String dictionaryPath = null;
+ ColumnIdentifier columnIdentifier = carbonDimension.getColumnIdentifier();
if (null != tableProvider) {
CarbonTable carbonTable = tableProvider
.getCarbonTable(dictionarySourceAbsoluteTableIdentifier.getCarbonTableIdentifier());
dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties()
.get(CarbonCommonConstants.DICTIONARY_PATH);
+ if (null != carbonDimension.getColumnSchema().getParentColumnTableRelations() &&
+ carbonDimension.getColumnSchema().getParentColumnTableRelations().size() == 1) {
+ dictionarySourceAbsoluteTableIdentifier =
+ QueryUtil.getTableIdentifierForColumn(carbonDimension,
+ carbonTable.getAbsoluteTableIdentifier());
+ columnIdentifier = new ColumnIdentifier(
+ carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(),
+ carbonDimension.getColumnProperties(), carbonDimension.getDataType());
+ } else {
+ dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
+ }
}
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier,
- carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), dictionaryPath);
+ columnIdentifier, carbonDimension.getDataType(), dictionaryPath);
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
cacheProvider.createCache(CacheType.FORWARD_DICTIONARY);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 7157b8a..fdeb2bd 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -219,6 +219,17 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
}
}
+ test("test if pre-agg table is hit with filter condition") {
+ sql("drop table if exists filtertable")
+ sql("CREATE TABLE filtertable(id int, name string, city string, age string) STORED BY" +
+ " 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='name,age')")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table filtertable")
+ sql("create datamap agg9 on table filtertable using 'preaggregate' as select name, age, sum(age) from filtertable group by name, age")
+ val df = sql("select name, sum(age) from filtertable where age = '29' group by name, age")
+ preAggTableValidator(df.queryExecution.analyzed, "filtertable_agg9")
+ checkAnswer(df, Row("vishal", 29))
+ }
+
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")
[16/50] [abbrv] carbondata git commit: [CARBONDATA-1858][PARTITION]
Support querying data from partition table.
Posted by ja...@apache.org.
[CARBONDATA-1858][PARTITION] Support querying data from partition table.
In case of partition table first, use sessioncatalog to prune the partitions. With the partition information, datamap should read partition.map file to get the index file and corresponding blocklets to prune
This closes #1672
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b8a02f39
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b8a02f39
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b8a02f39
Branch: refs/heads/fgdatamap
Commit: b8a02f391d9ff543abb54c7b58733cf9a4d7beaf
Parents: e6497e1
Author: ravipesala <ra...@gmail.com>
Authored: Mon Dec 18 22:37:59 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Dec 19 23:17:55 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/TableDataMap.java | 10 +-
.../carbondata/core/datamap/dev/DataMap.java | 11 +
.../indexstore/BlockletDataMapIndexStore.java | 37 ++--
.../blockletindex/BlockletDataMap.java | 117 +++++++---
.../blockletindex/BlockletDataMapModel.java | 11 +-
.../core/metadata/PartitionMapFileStore.java | 60 ++++--
.../apache/carbondata/core/util/CarbonUtil.java | 14 +-
.../core/util/path/CarbonTablePath.java | 2 +-
.../carbondata/core/util/CarbonUtilTest.java | 4 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../hadoop/api/CarbonTableInputFormat.java | 32 ++-
.../hadoop/api/DistributableDataMapFormat.java | 11 +-
.../StandardPartitionTableQueryTestCase.scala | 216 +++++++++++++++++++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 6 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 18 ++
.../sql/CarbonDatasourceHadoopRelation.scala | 7 +-
.../management/CarbonInsertIntoCommand.scala | 24 +--
.../management/CarbonLoadDataCommand.scala | 168 ++++++++-------
.../table/CarbonCreateTableCommand.scala | 5 +-
.../datasources/CarbonFileFormat.scala | 6 +-
.../strategy/CarbonLateDecodeStrategy.scala | 81 +++++--
.../spark/sql/optimizer/CarbonFilters.scala | 18 +-
22 files changed, 647 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 1d5c978..61d2243 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -64,14 +64,14 @@ public final class TableDataMap implements OperationEventListener {
* @param filterExp
* @return
*/
- public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp)
- throws IOException {
+ public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp,
+ List<String> partitions) throws IOException {
List<ExtendedBlocklet> blocklets = new ArrayList<>();
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp));
+ pruneBlocklets.addAll(dataMap.prune(filterExp, partitions));
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -118,12 +118,12 @@ public final class TableDataMap implements OperationEventListener {
* @return
*/
public List<ExtendedBlocklet> prune(DataMapDistributable distributable,
- FilterResolverIntf filterExp) throws IOException {
+ FilterResolverIntf filterExp, List<String> partitions) throws IOException {
List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
List<Blocklet> blocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
for (DataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(filterExp));
+ blocklets.addAll(dataMap.prune(filterExp, partitions));
}
for (Blocklet blocklet: blocklets) {
ExtendedBlocklet detailedBlocklet =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index e7c30a9..16be1ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -42,6 +42,17 @@ public interface DataMap {
*/
List<Blocklet> prune(FilterResolverIntf filterExp);
+ // TODO Move this method to Abstract class
+ /**
+ * Prune the datamap with filter expression and partition information. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions);
+
+ // TODO Move this method to Abstract class
/**
* Validate whether the current segment needs to be fetching the required data
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 4d90a08..59af50b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapModel;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.PartitionMapFileStore;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
@@ -71,12 +72,14 @@ public class BlockletDataMapIndexStore
BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
if (dataMap == null) {
try {
+ String segmentPath = CarbonTablePath
+ .getSegmentPath(identifier.getAbsoluteTableIdentifier().getTablePath(),
+ identifier.getSegmentId());
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
- indexFileStore.readAllIIndexOfSegment(
- CarbonTablePath.getSegmentPath(
- identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId()));
- dataMap = loadAndGetDataMap(identifier, indexFileStore);
+ indexFileStore.readAllIIndexOfSegment(segmentPath);
+ PartitionMapFileStore partitionFileStore = new PartitionMapFileStore();
+ partitionFileStore.readAllPartitionsOfSegment(segmentPath);
+ dataMap = loadAndGetDataMap(identifier, indexFileStore, partitionFileStore);
} catch (MemoryException e) {
LOGGER.error("memory exception when loading datamap: " + e.getMessage());
throw new RuntimeException(e.getMessage(), e);
@@ -102,18 +105,26 @@ public class BlockletDataMapIndexStore
}
if (missedIdentifiers.size() > 0) {
Map<String, SegmentIndexFileStore> segmentIndexFileStoreMap = new HashMap<>();
+ Map<String, PartitionMapFileStore> partitionFileStoreMap = new HashMap<>();
for (TableBlockIndexUniqueIdentifier identifier: missedIdentifiers) {
SegmentIndexFileStore indexFileStore =
segmentIndexFileStoreMap.get(identifier.getSegmentId());
+ PartitionMapFileStore partitionFileStore =
+ partitionFileStoreMap.get(identifier.getSegmentId());
+ String segmentPath = CarbonTablePath
+ .getSegmentPath(identifier.getAbsoluteTableIdentifier().getTablePath(),
+ identifier.getSegmentId());
if (indexFileStore == null) {
- String segmentPath = CarbonTablePath
- .getSegmentPath(identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId());
indexFileStore = new SegmentIndexFileStore();
indexFileStore.readAllIIndexOfSegment(segmentPath);
segmentIndexFileStoreMap.put(identifier.getSegmentId(), indexFileStore);
}
- blockletDataMaps.add(loadAndGetDataMap(identifier, indexFileStore));
+ if (partitionFileStore == null) {
+ partitionFileStore = new PartitionMapFileStore();
+ partitionFileStore.readAllPartitionsOfSegment(segmentPath);
+ partitionFileStoreMap.put(identifier.getSegmentId(), partitionFileStore);
+ }
+ blockletDataMaps.add(loadAndGetDataMap(identifier, indexFileStore, partitionFileStore));
}
}
} catch (Throwable e) {
@@ -160,7 +171,8 @@ public class BlockletDataMapIndexStore
*/
private BlockletDataMap loadAndGetDataMap(
TableBlockIndexUniqueIdentifier identifier,
- SegmentIndexFileStore indexFileStore)
+ SegmentIndexFileStore indexFileStore,
+ PartitionMapFileStore partitionFileStore)
throws IOException, MemoryException {
String uniqueTableSegmentIdentifier =
identifier.getUniqueTableSegmentIdentifier();
@@ -168,11 +180,12 @@ public class BlockletDataMapIndexStore
if (lock == null) {
lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
}
- BlockletDataMap dataMap = null;
+ BlockletDataMap dataMap;
synchronized (lock) {
dataMap = new BlockletDataMap();
dataMap.init(new BlockletDataMapModel(identifier.getFilePath(),
- indexFileStore.getFileData(identifier.getCarbonIndexFileName())));
+ indexFileStore.getFileData(identifier.getCarbonIndexFileName()),
+ partitionFileStore.getPartitions(identifier.getCarbonIndexFileName())));
lruCache.put(identifier.getUniqueTableSegmentIdentifier(), dataMap,
dataMap.getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 1dc9b7a..70bae32 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -99,7 +99,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
private UnsafeMemoryDMStore unsafeMemoryDMStore;
- private UnsafeMemoryDMStore unsafeMemoryTaskMinMaxDMStore;
+ private UnsafeMemoryDMStore unsafeMemorySummaryDMStore;
private SegmentProperties segmentProperties;
@@ -113,13 +113,14 @@ public class BlockletDataMap implements DataMap, Cacheable {
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
List<DataFileFooter> indexInfo = fileFooterConverter
.getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
+ DataMapRowImpl summaryRow = null;
for (DataFileFooter fileFooter : indexInfo) {
List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
if (segmentProperties == null) {
columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
createSchema(segmentProperties);
- createTaskMinMaxSchema(segmentProperties);
+ createSummarySchema(segmentProperties, blockletDataMapInfo.getPartitions());
}
TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
if (fileFooter.getBlockletList() == null || fileFooter.getBlockletList().size() == 0) {
@@ -128,28 +129,28 @@ public class BlockletDataMap implements DataMap, Cacheable {
fileFooter = CarbonUtil.readMetadatFile(blockInfo);
}
- loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath());
+ summaryRow = loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow);
}
if (unsafeMemoryDMStore != null) {
unsafeMemoryDMStore.finishWriting();
}
- if (null != unsafeMemoryTaskMinMaxDMStore) {
- unsafeMemoryTaskMinMaxDMStore.finishWriting();
+ if (null != unsafeMemorySummaryDMStore) {
+ addTaskSummaryRowToUnsafeMemoryStore(summaryRow, blockletDataMapInfo.getPartitions());
+ unsafeMemorySummaryDMStore.finishWriting();
}
LOGGER.info(
"Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (
System.currentTimeMillis() - startTime));
}
- private void loadToUnsafe(DataFileFooter fileFooter, SegmentProperties segmentProperties,
- String filePath) {
+ private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
+ SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow) {
int[] minMaxLen = segmentProperties.getColumnsValueSize();
List<BlockletInfo> blockletList = fileFooter.getBlockletList();
CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
- DataMapRow taskMinMaxRow = null;
// Add one row to maintain task level min max for segment pruning
- if (!blockletList.isEmpty()) {
- taskMinMaxRow = new DataMapRowImpl(unsafeMemoryTaskMinMaxDMStore.getSchema());
+ if (!blockletList.isEmpty() && summaryRow == null) {
+ summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
}
for (int index = 0; index < blockletList.size(); index++) {
DataMapRow row = new DataMapRowImpl(schema);
@@ -164,16 +165,16 @@ public class BlockletDataMap implements DataMap, Cacheable {
byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
// compute and set task level min values
- addTaskMinMaxValues(taskMinMaxRow, minMaxLen,
- unsafeMemoryTaskMinMaxDMStore.getSchema()[taskMinMaxOrdinal], minValues,
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues,
TASK_MIN_VALUES_INDEX, true);
ordinal++;
taskMinMaxOrdinal++;
byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
// compute and set task level max values
- addTaskMinMaxValues(taskMinMaxRow, minMaxLen,
- unsafeMemoryTaskMinMaxDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
TASK_MAX_VALUES_INDEX, false);
ordinal++;
@@ -205,17 +206,31 @@ public class BlockletDataMap implements DataMap, Cacheable {
throw new RuntimeException(e);
}
}
- // write the task level min/max row to unsafe memory store
- if (null != taskMinMaxRow) {
- addTaskMinMaxRowToUnsafeMemoryStore(taskMinMaxRow);
- }
+
+ return summaryRow;
}
- private void addTaskMinMaxRowToUnsafeMemoryStore(DataMapRow taskMinMaxRow) {
- try {
- unsafeMemoryTaskMinMaxDMStore.addIndexRowToUnsafe(taskMinMaxRow);
- } catch (Exception e) {
- throw new RuntimeException(e);
+ private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow,
+ List<String> partitions) {
+ // write the task summary info to unsafe memory store
+ if (null != summaryRow) {
+ if (partitions != null && partitions.size() > 0) {
+ CarbonRowSchema[] minSchemas =
+ ((CarbonRowSchema.StructCarbonRowSchema) unsafeMemorySummaryDMStore.getSchema()[2])
+ .getChildSchemas();
+ DataMapRow partitionRow = new DataMapRowImpl(minSchemas);
+ for (int i = 0; i < partitions.size(); i++) {
+ partitionRow
+ .setByteArray(partitions.get(i).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
+ i);
+ }
+ summaryRow.setRow(partitionRow, 2);
+ }
+ try {
+ unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
@@ -379,10 +394,21 @@ public class BlockletDataMap implements DataMap, Cacheable {
new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
}
- private void createTaskMinMaxSchema(SegmentProperties segmentProperties) throws MemoryException {
+ private void createSummarySchema(SegmentProperties segmentProperties, List<String> partitions)
+ throws MemoryException {
List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>(2);
getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
- unsafeMemoryTaskMinMaxDMStore = new UnsafeMemoryDMStore(
+ if (partitions != null && partitions.size() > 0) {
+ CarbonRowSchema[] mapSchemas = new CarbonRowSchema[partitions.size()];
+ for (int i = 0; i < mapSchemas.length; i++) {
+ mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
+ }
+ CarbonRowSchema mapSchema =
+ new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
+ mapSchemas);
+ taskMinMaxSchemas.add(mapSchema);
+ }
+ unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
}
@@ -412,8 +438,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
public boolean isScanRequired(FilterResolverIntf filterExp) {
FilterExecuter filterExecuter =
FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- for (int i = 0; i < unsafeMemoryTaskMinMaxDMStore.getRowCount(); i++) {
- DataMapRow unsafeRow = unsafeMemoryTaskMinMaxDMStore.getUnsafeRow(i);
+ for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
boolean isScanRequired = FilterExpressionProcessor
.isScanRequired(filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
@@ -491,6 +517,20 @@ public class BlockletDataMap implements DataMap, Cacheable {
return blocklets;
}
+ @Override public List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions) {
+ List<String> storedPartitions = getPartitions();
+ if (storedPartitions != null && storedPartitions.size() > 0 && filterExp != null) {
+ boolean found = false;
+ if (partitions != null && partitions.size() > 0) {
+ found = partitions.containsAll(storedPartitions);
+ }
+ if (!found) {
+ return new ArrayList<>();
+ }
+ }
+ return prune(filterExp);
+ }
+
/**
* select the blocks based on column min and max value
*
@@ -664,6 +704,19 @@ public class BlockletDataMap implements DataMap, Cacheable {
return dataMapRow;
}
+ private List<String> getPartitions() {
+ List<String> partitions = new ArrayList<>();
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ if (unsafeRow.getColumnCount() > 2) {
+ DataMapRow row = unsafeRow.getRow(2);
+ for (int i = 0; i < row.getColumnCount(); i++) {
+ partitions.add(
+ new String(row.getByteArray(i), CarbonCommonConstants.DEFAULT_CHARSET_CLASS));
+ }
+ }
+ return partitions;
+ }
+
@Override
public void clear() {
if (unsafeMemoryDMStore != null) {
@@ -672,9 +725,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
segmentProperties = null;
}
// clear task min/max unsafe memory
- if (null != unsafeMemoryTaskMinMaxDMStore) {
- unsafeMemoryTaskMinMaxDMStore.freeMemory();
- unsafeMemoryTaskMinMaxDMStore = null;
+ if (null != unsafeMemorySummaryDMStore) {
+ unsafeMemorySummaryDMStore.freeMemory();
+ unsafeMemorySummaryDMStore = null;
}
}
@@ -694,8 +747,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
if (unsafeMemoryDMStore != null) {
memoryUsed += unsafeMemoryDMStore.getMemoryUsed();
}
- if (null != unsafeMemoryTaskMinMaxDMStore) {
- memoryUsed += unsafeMemoryTaskMinMaxDMStore.getMemoryUsed();
+ if (null != unsafeMemorySummaryDMStore) {
+ memoryUsed += unsafeMemorySummaryDMStore.getMemoryUsed();
}
return memoryUsed;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index 7ffa32d..c98ef33 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -16,18 +16,27 @@
*/
package org.apache.carbondata.core.indexstore.blockletindex;
+import java.util.List;
+
import org.apache.carbondata.core.datamap.dev.DataMapModel;
public class BlockletDataMapModel extends DataMapModel {
private byte[] fileData;
- public BlockletDataMapModel(String filePath, byte[] fileData) {
+ private List<String> partitions;
+
+ public BlockletDataMapModel(String filePath, byte[] fileData, List<String> partitions) {
super(filePath);
this.fileData = fileData;
+ this.partitions = partitions;
}
public byte[] getFileData() {
return fileData;
}
+
+ public List<String> getPartitions() {
+ return partitions;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 4b58e75..853c729 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -46,6 +46,7 @@ import com.google.gson.Gson;
*/
public class PartitionMapFileStore {
+ private Map<String, List<String>> partitionMap = new HashMap<>();
/**
* Write partitionmapp file to the segment folder with indexfilename and corresponding partitions.
*
@@ -105,33 +106,38 @@ public class PartitionMapFileStore {
* @throws IOException
*/
public void mergePartitionMapFiles(String segmentPath) throws IOException {
+ CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
+ if (partitionFiles != null && partitionFiles.length > 1) {
+ PartitionMapper partitionMapper = null;
+ for (CarbonFile file : partitionFiles) {
+ PartitionMapper localMapper = readPartitionMap(file.getAbsolutePath());
+ if (partitionMapper == null && localMapper != null) {
+ partitionMapper = localMapper;
+ }
+ if (localMapper != null) {
+ partitionMapper = partitionMapper.merge(localMapper);
+ }
+ }
+ if (partitionMapper != null) {
+ String path = segmentPath + "/" + "mergedpartitions" + CarbonTablePath.PARTITION_MAP_EXT;
+ writePartitionFile(partitionMapper, path);
+ for (CarbonFile file : partitionFiles) {
+ FileFactory.deleteAllCarbonFilesOfDir(file);
+ }
+ }
+ }
+ }
+
+ private CarbonFile[] getPartitionFiles(String segmentPath) {
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
if (carbonFile.exists()) {
- CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ return carbonFile.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
}
});
- if (carbonFiles != null && carbonFiles.length > 1) {
- PartitionMapper partitionMapper = null;
- for (CarbonFile file : carbonFiles) {
- PartitionMapper localMapper = readPartitionMap(file.getAbsolutePath());
- if (partitionMapper == null && localMapper != null) {
- partitionMapper = localMapper;
- }
- if (localMapper != null) {
- partitionMapper = partitionMapper.merge(localMapper);
- }
- }
- if (partitionMapper != null) {
- String path = segmentPath + "/" + "mergedpartitions" + CarbonTablePath.PARTITION_MAP_EXT;
- writePartitionFile(partitionMapper, path);
- for (CarbonFile file : carbonFiles) {
- FileFactory.deleteAllCarbonFilesOfDir(file);
- }
- }
- }
}
+ return null;
}
/**
@@ -167,6 +173,20 @@ public class PartitionMapFileStore {
return partitionMapper;
}
+ public void readAllPartitionsOfSegment(String segmentPath) {
+ CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
+ if (partitionFiles != null && partitionFiles.length > 0) {
+ for (CarbonFile file : partitionFiles) {
+ PartitionMapper partitionMapper = readPartitionMap(file.getAbsolutePath());
+ partitionMap.putAll(partitionMapper.getPartitionMap());
+ }
+ }
+ }
+
+ public List<String> getPartitions(String indexFileName) {
+ return partitionMap.get(indexFileName);
+ }
+
public static class PartitionMapper implements Serializable {
private static final long serialVersionUID = 3582245668420401089L;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 910efea..bd01772 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1391,23 +1391,23 @@ public final class CarbonUtil {
}
/**
- * Below method will be used to get the list of segment in
+ * Below method will be used to get the list of values in
* comma separated string format
*
- * @param segmentList
+ * @param values
* @return comma separated segment string
*/
- public static String getSegmentString(List<String> segmentList) {
- if (segmentList.isEmpty()) {
+ public static String convertToString(List<String> values) {
+ if (values == null || values.isEmpty()) {
return "";
}
StringBuilder segmentStringbuilder = new StringBuilder();
- for (int i = 0; i < segmentList.size() - 1; i++) {
- String segmentNo = segmentList.get(i);
+ for (int i = 0; i < values.size() - 1; i++) {
+ String segmentNo = values.get(i);
segmentStringbuilder.append(segmentNo);
segmentStringbuilder.append(",");
}
- segmentStringbuilder.append(segmentList.get(segmentList.size() - 1));
+ segmentStringbuilder.append(values.get(values.size() - 1));
return segmentStringbuilder.toString();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 6be1f4c..c33c0a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -42,10 +42,10 @@ public class CarbonTablePath extends Path {
private static final String FACT_DIR = "Fact";
private static final String SEGMENT_PREFIX = "Segment_";
private static final String PARTITION_PREFIX = "Part";
- private static final String CARBON_DATA_EXT = ".carbondata";
private static final String DATA_PART_PREFIX = "part-";
private static final String BATCH_PREFIX = "_batchno";
+ public static final String CARBON_DATA_EXT = ".carbondata";
public static final String INDEX_FILE_EXT = ".carbonindex";
public static final String MERGE_INDEX_FILE_EXT = ".carbonindexmerge";
public static final String PARTITION_MAP_EXT = ".partitionmap";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index e653137..2296641 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -716,13 +716,13 @@ public class CarbonUtilTest {
List<String> list = new ArrayList<>();
list.add("1");
list.add("2");
- String segments = CarbonUtil.getSegmentString(list);
+ String segments = CarbonUtil.convertToString(list);
assertEquals(segments, "1,2");
}
@Test public void testToGetSegmentStringWithEmptySegmentList() {
List<String> list = new ArrayList<>();
- String segments = CarbonUtil.getSegmentString(list);
+ String segments = CarbonUtil.convertToString(list);
assertEquals(segments, "");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 1803a12..3e8ede2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -299,14 +299,14 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
*/
public static void setSegmentsToAccess(Configuration configuration, List<String> validSegments) {
configuration
- .set(CarbonInputFormat.INPUT_SEGMENT_NUMBERS, CarbonUtil.getSegmentString(validSegments));
+ .set(CarbonInputFormat.INPUT_SEGMENT_NUMBERS, CarbonUtil.convertToString(validSegments));
}
/**
* Set list of files to access
*/
public static void setFilesToAccess(Configuration configuration, List<String> validFiles) {
- configuration.set(CarbonInputFormat.INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
+ configuration.set(CarbonInputFormat.INPUT_FILES, CarbonUtil.convertToString(validFiles));
}
private static AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 9e8e22d..e60a588 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -127,6 +127,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
private static final String DATA_MAP_DSTR = "mapreduce.input.carboninputformat.datamapdstr";
public static final String DATABASE_NAME = "mapreduce.input.carboninputformat.databaseName";
public static final String TABLE_NAME = "mapreduce.input.carboninputformat.tableName";
+ public static final String PARTITIONS_TO_PRUNE =
+ "mapreduce.input.carboninputformat.partitions.to.prune";
// a cache for carbon table, it will be used in task side
private CarbonTable carbonTable;
@@ -257,7 +259,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* Set list of segments to access
*/
public static void setSegmentsToAccess(Configuration configuration, List<String> validSegments) {
- configuration.set(INPUT_SEGMENT_NUMBERS, CarbonUtil.getSegmentString(validSegments));
+ configuration.set(INPUT_SEGMENT_NUMBERS, CarbonUtil.convertToString(validSegments));
}
/**
@@ -290,10 +292,28 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
/**
+ * set list of partitions to prune
+ */
+ public static void setPartitionsToPrune(Configuration configuration, List<String> partitions) {
+ configuration.set(
+ CarbonTableInputFormat.PARTITIONS_TO_PRUNE, CarbonUtil.convertToString(partitions));
+ }
+
+ /**
+ * get list of partitions to prune
+ */
+ public static List<String> getPartitionsToPrune(Configuration configuration) {
+ String partitionString = configuration.get(PARTITIONS_TO_PRUNE, "");
+ if (partitionString.trim().isEmpty()) {
+ return null;
+ }
+ return Arrays.asList(partitionString.split(","));
+ }
+ /**
* Set list of files to access
*/
public static void setFilesToAccess(Configuration configuration, List<String> validFiles) {
- configuration.set(INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
+ configuration.set(INPUT_FILES, CarbonUtil.convertToString(validFiles));
}
public AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
@@ -710,14 +730,16 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
.getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME,
BlockletDataMapFactory.class.getName());
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
+ List<String> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
if (dataMapJob != null) {
DistributableDataMapFormat datamapDstr =
new DistributableDataMapFormat(absoluteTableIdentifier, BlockletDataMap.NAME,
- segmentIds, BlockletDataMapFactory.class.getName());
+ segmentIds, partitionsToPrune,
+ BlockletDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
} else {
- prunedBlocklets = blockletMap.prune(segmentIds, resolver);
+ prunedBlocklets = blockletMap.prune(segmentIds, resolver, partitionsToPrune);
}
List<org.apache.carbondata.hadoop.CarbonInputSplit> resultFilterredBlocks = new ArrayList<>();
@@ -908,7 +930,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// TODO: currently only batch segment is supported, add support for streaming table
List<String> filteredSegment = getFilteredSegment(job, allSegments.getValidSegments());
- List<ExtendedBlocklet> blocklets = blockletMap.prune(filteredSegment, null);
+ List<ExtendedBlocklet> blocklets = blockletMap.prune(filteredSegment, null, null);
for (ExtendedBlocklet blocklet : blocklets) {
String blockName = blocklet.getPath();
blockName = CarbonTablePath.getCarbonDataFileName(blockName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
index 1b875bc..96eec6f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
@@ -53,12 +53,15 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
private String className;
+ private List<String> partitions;
+
public DistributableDataMapFormat(AbsoluteTableIdentifier identifier,
- String dataMapName, List<String> validSegments, String className) {
+ String dataMapName, List<String> validSegments, List<String> partitions, String className) {
this.identifier = identifier;
this.dataMapName = dataMapName;
this.validSegments = validSegments;
this.className = className;
+ this.partitions = partitions;
}
public static void setFilterExp(Configuration configuration, FilterResolverIntf filterExp)
@@ -102,9 +105,9 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
TableDataMap dataMap = DataMapStoreManager.getInstance()
.getDataMap(identifier, distributable.getDataMapName(),
distributable.getDataMapFactoryClass());
- blockletIterator =
- dataMap.prune(distributable, getFilterExp(taskAttemptContext.getConfiguration()))
- .iterator();
+ blockletIterator = dataMap.prune(
+ distributable, getFilterExp(taskAttemptContext.getConfiguration()), partitions)
+ .iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
new file mode 100644
index 0000000..570951a
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.execution.BatchedDataSourceScanExec
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.spark.rdd.CarbonScanRDD
+
+class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ test("querying on partition table for int partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ val frame = sql(
+ "select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno," +
+ " deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary " +
+ "from partitionone where empno=11 order by empno")
+
+ verifyPartitionInfo(frame, Seq("empno=11"))
+
+ checkAnswer(frame,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where empno=11 order by empno"))
+
+ }
+
+ test("querying on partition table for string partition column") {
+ sql(
+ """
+ | CREATE TABLE partitiontwo (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwo OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ val frame = sql(
+ "select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno," +
+ " deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary " +
+ "from partitiontwo where deptname='network' and projectcode=928478")
+ verifyPartitionInfo(frame, Seq("deptname=network"))
+
+ val frame1 = sql(
+ "select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno," +
+ " deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary " +
+ "from partitiontwo where projectcode=928478")
+ checkAnswer(frame1,
+ sql( "select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno," +
+ " deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary " +
+ "from originTable where projectcode=928478"))
+ verifyPartitionInfo(frame1, Seq("deptname=network","deptname=security","deptname=protocol","deptname=Learning","deptname=configManagement"))
+
+ val frame2 = sql("select distinct deptname from partitiontwo")
+
+ verifyPartitionInfo(frame2, Seq("deptname=network","deptname=security","deptname=protocol","deptname=Learning","deptname=configManagement"))
+
+ checkAnswer(frame,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where where deptname='network' and projectcode=928478 order by empno"))
+
+ }
+
+ test("querying on partition table for more partition columns") {
+ sql(
+ """
+ | CREATE TABLE partitionmany (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ val frame = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmany where doj='2007-01-17 00:00:00'")
+ verifyPartitionInfo(frame, Seq("deptname=network","doj=2007-01-17 00:00:00","projectcode=928478"))
+ checkAnswer(frame,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where doj='2007-01-17 00:00:00'"))
+
+ }
+
+ test("querying on partition table for date partition column") {
+ sql(
+ """
+ | CREATE TABLE partitiondate (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,doj Timestamp,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondate OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val frame = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiondate where projectenddate = cast('2016-11-30' as date)")
+ verifyPartitionInfo(frame, Seq("projectenddate=2016-11-30"))
+ checkAnswer(frame,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where projectenddate = cast('2016-11-30' as date)"))
+
+ }
+
+ test("querying on partition table for date partition column on insert query") {
+ sql(
+ """
+ | CREATE TABLE partitiondateinsert (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date,doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ val frame = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiondateinsert where projectenddate = cast('2016-11-30' as date)")
+ verifyPartitionInfo(frame, Seq("projectenddate=2016-11-30","doj=2015-12-01 00:00:00"))
+ checkAnswer(frame,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where projectenddate = cast('2016-11-30' as date)"))
+
+ val frame1 = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiondateinsert where doj>'2006-01-17 00:00:00'")
+ verifyPartitionInfo(frame1,
+ Seq("projectenddate=2016-06-29" ,
+ "doj=2010-12-29 00:00:00" ,
+ "doj=2015-12-01 00:00:00" ,
+ "projectenddate=2016-11-12" ,
+ "projectenddate=2016-12-29" ,
+ "doj=2011-11-09 00:00:00" ,
+ "doj=2009-07-07 00:00:00" ,
+ "projectenddate=2016-05-29" ,
+ "doj=2012-10-14 00:00:00" ,
+ "projectenddate=2016-11-30" ,
+ "projectenddate=2016-11-15",
+ "doj=2015-05-12 00:00:00" ,
+ "doj=2013-09-22 00:00:00" ,
+ "doj=2008-05-29 00:00:00" ,
+ "doj=2014-08-15 00:00:00",
+ "projectenddate=2016-12-30"))
+ checkAnswer(frame1,
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where doj>'2006-01-17 00:00:00'"))
+
+ }
+
+
+ private def verifyPartitionInfo(frame: DataFrame, partitionNames: Seq[String]) = {
+ val plan = frame.queryExecution.sparkPlan
+ val scanRDD = plan collect {
+ case b: BatchedDataSourceScanExec if b.rdd.isInstanceOf[CarbonScanRDD] => b.rdd
+ .asInstanceOf[CarbonScanRDD]
+ }
+ assert(scanRDD.nonEmpty)
+ assert(!partitionNames.map(f => scanRDD.head.partitionNames.exists(_.equals(f))).exists(!_))
+ }
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitiontwo")
+ sql("drop table if exists partitionmany")
+ sql("drop table if exists partitiondate")
+ sql("drop table if exists partitiondateinsert")
+ sql("drop table if exists staticpartitionone")
+ sql("drop table if exists singlepasspartitionone")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index cc7f757..e58bfd4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -61,7 +61,8 @@ class CarbonScanRDD(
identifier: AbsoluteTableIdentifier,
@transient serializedTableInfo: Array[Byte],
@transient tableInfo: TableInfo,
- inputMetricsStats: InitInputMetrics)
+ inputMetricsStats: InitInputMetrics,
+ @transient val partitionNames: Seq[String])
extends CarbonRDDWithTableInfo[InternalRow](sc, Nil, serializedTableInfo) {
private val queryId = sparkContext.getConf.get("queryId", System.nanoTime() + "")
@@ -332,6 +333,9 @@ class CarbonScanRDD(
CarbonTableInputFormat.setTableInfo(conf, tableInfo)
CarbonTableInputFormat.setDatabaseName(conf, tableInfo.getDatabaseName)
CarbonTableInputFormat.setTableName(conf, tableInfo.getFactTable.getTableName)
+ if (partitionNames != null) {
+ CarbonTableInputFormat.setPartitionsToPrune(conf, partitionNames.asJava)
+ }
createInputFormat(conf)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 653e6f3..f78412b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -22,8 +22,10 @@ import java.text.SimpleDateFormat
import java.util
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.command.DataTypeInfo
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
@@ -152,6 +154,22 @@ object CarbonScalaUtil {
}
}
+ def convertToUTF8String(value: String,
+ dataType: DataType,
+ timeStampFormat: SimpleDateFormat,
+ dateFormat: SimpleDateFormat): UTF8String = {
+ dataType match {
+ case TimestampType =>
+ UTF8String.fromString(
+ DateTimeUtils.timestampToString(timeStampFormat.parse(value).getTime * 1000))
+ case DateType =>
+ UTF8String.fromString(
+ DateTimeUtils.dateToString(
+ (dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt))
+ case _ => UTF8String.fromString(value)
+ }
+ }
+
/**
* This method will validate a column for its data type and check whether the column data type
* can be modified and update if conditions are met
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 19b96a8..148fca8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -65,7 +65,9 @@ case class CarbonDatasourceHadoopRelation(
override def schema: StructType = tableSchema.getOrElse(carbonRelation.schema)
- def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[InternalRow] = {
+ def buildScan(requiredColumns: Array[String],
+ filters: Array[Filter],
+ partitions: Seq[String]): RDD[InternalRow] = {
val filterExpression: Option[Expression] = filters.flatMap { filter =>
CarbonFilters.createCarbonFilter(schema, filter)
}.reduceOption(new AndExpression(_, _))
@@ -80,7 +82,8 @@ case class CarbonDatasourceHadoopRelation(
identifier,
carbonTable.getTableInfo.serialize(),
carbonTable.getTableInfo,
- inputMetricsStats)
+ inputMetricsStats,
+ partitions)
}
override def unhandledFilters(filters: Array[Filter]): Array[Filter] = new Array[Filter](0)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 5142ef2..810b10f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -34,18 +34,18 @@ case class CarbonInsertIntoCommand(
val df = Dataset.ofRows(sparkSession, child)
val header = relation.tableSchema.get.fields.map(_.name).mkString(",")
val load = CarbonLoadDataCommand(
- Some(relation.carbonRelation.databaseName),
- relation.carbonRelation.tableName,
- null,
- Seq(),
- scala.collection.immutable.Map("fileheader" -> header),
- overwrite,
- null,
- Some(df),
- None,
- None,
- Map.empty,
- partition).run(sparkSession)
+ databaseNameOp = Some(relation.carbonRelation.databaseName),
+ tableName = relation.carbonRelation.tableName,
+ factPathFromUser = null,
+ dimFilesPath = Seq(),
+ options = scala.collection.immutable.Map("fileheader" -> header),
+ isOverwriteTable = overwrite,
+ inputSqlString = null,
+ dataFrame = Some(df),
+ updateModel = None,
+ tableInfoOp = None,
+ internalOptions = Map.empty,
+ partition = partition).run(sparkSession)
// updating relation metadata. This is in case of auto detect high cardinality
relation.carbonRelation.metaData =
CarbonSparkUtil.createSparkMeta(relation.carbonRelation.carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index c94f698..a883735 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -34,12 +34,13 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.catalog.{CatalogRelation, CatalogTable}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.types.{StringType, StructField, StructType}
+import org.apache.spark.sql.types.{StringType, StructField, StructType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
@@ -363,7 +364,7 @@ case class CarbonLoadDataCommand(
if (carbonTable.isHivePartitionTable) {
try {
- loadStandardPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
+ loadDataWithPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
} finally {
server match {
case Some(dictServer) =>
@@ -378,7 +379,8 @@ case class CarbonLoadDataCommand(
}
}
} else {
- CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
+ CarbonDataRDDFactory.loadCarbonData(
+ sparkSession.sqlContext,
carbonLoadModel,
columnar,
partitionStatus,
@@ -417,9 +419,10 @@ case class CarbonLoadDataCommand(
dictionaryDataFrame)
}
if (table.isHivePartitionTable) {
- loadStandardPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
+ loadDataWithPartition(sparkSession, carbonLoadModel, hadoopConf, loadDataFrame)
} else {
- CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
+ CarbonDataRDDFactory.loadCarbonData(
+ sparkSession.sqlContext,
carbonLoadModel,
columnar,
partitionStatus,
@@ -432,7 +435,17 @@ case class CarbonLoadDataCommand(
}
}
- private def loadStandardPartition(sparkSession: SparkSession,
+ /**
+ * Loads the data in a hive partition way. This method uses InsertIntoTable command to load data
+ * into partitoned data. The table relation would be converted to HadoopFSRelation to let spark
+ * handling the partitioning.
+ * @param sparkSession
+ * @param carbonLoadModel
+ * @param hadoopConf
+ * @param dataFrame
+ * @return
+ */
+ private def loadDataWithPartition(sparkSession: SparkSession,
carbonLoadModel: CarbonLoadModel,
hadoopConf: Configuration,
dataFrame: Option[DataFrame]) = {
@@ -443,15 +456,14 @@ case class CarbonLoadDataCommand(
val relation = logicalPlan.collect {
case l: LogicalRelation => l
case c: CatalogRelation => c
- }.head.asInstanceOf[LogicalPlan]
+ }.head
+ // Converts the data to carbon understandable format. The timestamp/date format data needs to
+ // converted to hive standard fomat to let spark understand the data to partition.
val query: LogicalPlan = if (dataFrame.isDefined) {
- val timeStampformatString = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ var timeStampformatString = CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
val timeStampFormat = new SimpleDateFormat(timeStampformatString)
- val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
- .CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ var dateFormatString = CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT
val dateFormat = new SimpleDateFormat(dateFormatString)
val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
@@ -479,31 +491,81 @@ case class CarbonLoadDataCommand(
LogicalRDD(attributes, rdd)(sparkSession)
} else {
+ var timeStampformatString = carbonLoadModel.getTimestampformat
+ if (timeStampformatString.isEmpty) {
+ timeStampformatString = carbonLoadModel.getDefaultTimestampFormat
+ }
+ val timeStampFormat = new SimpleDateFormat(timeStampformatString)
+ var dateFormatString = carbonLoadModel.getDateFormat
+ if (dateFormatString.isEmpty) {
+ dateFormatString = carbonLoadModel.getDefaultDateFormat
+ }
+ val dateFormat = new SimpleDateFormat(dateFormatString)
// input data from csv files. Convert to logical plan
CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
val jobConf = new JobConf(hadoopConf)
SparkHadoopUtil.get.addCredentials(jobConf)
- val rdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
- sparkSession.sparkContext,
- classOf[CSVInputFormat],
- classOf[NullWritable],
- classOf[StringArrayWritable],
- jobConf
- ).map(f => InternalRow.fromSeq(f._2.get().map(UTF8String.fromString)))
-
val attributes =
StructType(carbonLoadModel.getCsvHeaderColumns.map(
StructField(_, StringType))).toAttributes
+ val rowDataTypes = attributes.map { attribute =>
+ relation.output.find(_.name.equalsIgnoreCase(attribute.name)) match {
+ case Some(attr) => attr.dataType
+ case _ => StringType
+ }
+ }
+ val len = rowDataTypes.length
+ val rdd =
+ new NewHadoopRDD[NullWritable, StringArrayWritable](
+ sparkSession.sparkContext,
+ classOf[CSVInputFormat],
+ classOf[NullWritable],
+ classOf[StringArrayWritable],
+ jobConf).map{ case (key, value) =>
+ val data = new Array[Any](len)
+ var i = 0
+ while (i < len) {
+ // TODO find a way to avoid double conversion of date and time.
+ data(i) = CarbonScalaUtil.convertToUTF8String(
+ value.get()(i),
+ rowDataTypes(i),
+ timeStampFormat,
+ dateFormat)
+ i = i + 1
+ }
+ InternalRow.fromSeq(data)
+ }
+
// Only select the required columns
Project(relation.output.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get),
LogicalRDD(attributes, rdd)(sparkSession))
}
val convertRelation = relation match {
case l: LogicalRelation =>
- convertToLogicalRelation(l, isOverwriteTable, carbonLoadModel, sparkSession)
+ convertToLogicalRelation(
+ l.catalogTable.get,
+ l.output,
+ l.relation.sizeInBytes,
+ isOverwriteTable,
+ carbonLoadModel,
+ sparkSession)
case c: CatalogRelation =>
- convertToLogicalRelation(c, isOverwriteTable, carbonLoadModel, sparkSession)
+ val catalogTable = CarbonReflectionUtils.getFieldOfCatalogTable(
+ "tableMeta",
+ relation).asInstanceOf[CatalogTable]
+ // TODO need to find a way to avoid double lookup
+ val sizeInBytes =
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
+ catalogTable.identifier)(sparkSession).asInstanceOf[CarbonRelation].sizeInBytes
+ val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
+ convertToLogicalRelation(
+ catalogTable,
+ c.output,
+ sizeInBytes,
+ isOverwriteTable,
+ carbonLoadModel,
+ sparkSession)
}
Dataset.ofRows(
sparkSession,
@@ -516,74 +578,26 @@ case class CarbonLoadDataCommand(
}
private def convertToLogicalRelation(
- relation: LogicalRelation,
+ catalogTable: CatalogTable,
+ output: Seq[Attribute],
+ sizeInBytes: Long,
overWrite: Boolean,
loadModel: CarbonLoadModel,
sparkSession: SparkSession): LogicalRelation = {
- val catalogTable = relation.catalogTable.get
val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
val metastoreSchema = StructType(StructType.fromAttributes(
- relation.output).fields.map(_.copy(dataType = StringType)))
+ output).fields.map(_.copy(dataType = StringType)))
val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
val catalog = new CatalogFileIndex(
- sparkSession, catalogTable, relation.relation.sizeInBytes)
+ sparkSession, catalogTable, sizeInBytes)
if (lazyPruningEnabled) {
catalog
} else {
catalog.filterPartitions(Nil) // materialize all the partitions in memory
}
val partitionSchema =
- StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(f =>
- metastoreSchema.fields.find(_.name.equalsIgnoreCase(f.getColumnName))).map(_.get))
-
-
- val dataSchema =
- StructType(metastoreSchema
- .filterNot(field => partitionSchema.contains(field.name)))
- val options = new mutable.HashMap[String, String]()
- options ++= catalogTable.storage.properties
- options += (("overwrite", overWrite.toString))
- options += (("onepass", loadModel.getUseOnePass.toString))
- options += (("dicthost", loadModel.getDictionaryServerHost))
- options += (("dictport", loadModel.getDictionaryServerPort.toString))
- val hdfsRelation = HadoopFsRelation(
- location = catalog,
- partitionSchema = partitionSchema,
- dataSchema = dataSchema,
- bucketSpec = catalogTable.bucketSpec,
- fileFormat = new CarbonFileFormat,
- options = options.toMap)(sparkSession = sparkSession)
-
- CarbonReflectionUtils.getLogicalRelation(hdfsRelation,
- hdfsRelation.schema.toAttributes,
- Some(catalogTable))
- }
-
- private def convertToLogicalRelation(
- relation: CatalogRelation,
- overWrite: Boolean,
- loadModel: CarbonLoadModel,
- sparkSession: SparkSession): LogicalRelation = {
- val catalogTable =
- CarbonReflectionUtils.getFieldOfCatalogTable("tableMeta", relation).asInstanceOf[CatalogTable]
- val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
- val metastoreSchema = StructType(StructType.fromAttributes(
- relation.output).fields.map(_.copy(dataType = StringType)))
- val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
- // TODO nedd to find a way to avoid double lookup
- val sizeInBytes =
- CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
- catalogTable.identifier)(sparkSession).asInstanceOf[CarbonRelation].sizeInBytes
- val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
- if (lazyPruningEnabled) {
- catalog
- } else {
- catalog.filterPartitions(Nil) // materialize all the partitions in memory
- }
- val partitionSchema =
- StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(f =>
- metastoreSchema.fields.find(_.name.equalsIgnoreCase(f.getColumnName))).map(_.get))
-
+ StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(field =>
+ metastoreSchema.fields.find(_.name.equalsIgnoreCase(field.getColumnName))).map(_.get))
val dataSchema =
StructType(metastoreSchema
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index a42d974..f86b35f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -29,6 +29,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.exception.InvalidConfigurationException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.events.{CreateTablePostExecutionEvent, CreateTablePreExecutionEvent, OperationContext, OperationListenerBus}
@@ -94,7 +95,9 @@ case class CarbonCreateTableCommand(
val rawSchema = CarbonSparkUtil.getRawSchema(carbonRelation)
sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
val partitionInfo = tableInfo.getFactTable.getPartitionInfo
- val partitionString = if (partitionInfo != null) {
+ val partitionString =
+ if (partitionInfo != null &&
+ partitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
_.getColumnName).mkString(",")})"
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 99ce7fa..3eed726 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -72,7 +72,6 @@ with Serializable {
sparkSession.sessionState.conf.setConfString(
"spark.sql.sources.commitProtocolClass",
"org.apache.spark.sql.execution.datasources.CarbonSQLHadoopMapReduceCommitProtocol")
-
job.setOutputFormatClass(classOf[CarbonTableOutputFormat])
var table = CarbonEnv.getCarbonTable(
TableIdentifier(options("tableName"), options.get("dbName")))(sparkSession)
@@ -98,6 +97,9 @@ with Serializable {
model,
conf
)
+ // Set the standard date/time format which supported by spark/hive.
+ model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
model.setPartitionId("0")
model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
@@ -139,7 +141,7 @@ with Serializable {
}
override def getFileExtension(context: TaskAttemptContext): String = {
- ".carbondata"
+ CarbonTablePath.CARBON_DATA_EXT
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 7f9bdf7..1e71714 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.optimizer.CarbonDecoderRelation
+import org.apache.spark.sql.optimizer.{CarbonDecoderRelation, CarbonFilters}
import org.apache.spark.sql.sources.{BaseRelation, Filter}
import org.apache.spark.sql.types._
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
@@ -60,8 +60,8 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
l,
projects,
filters,
- (a, f, needDecoder) => toCatalystRDD(l, a, relation.buildScan(
- a.map(_.name).toArray, f), needDecoder)) :: Nil
+ (a, f, needDecoder, p) => toCatalystRDD(l, a, relation.buildScan(
+ a.map(_.name).toArray, f, p), needDecoder)) :: Nil
case CarbonDictionaryCatalystDecoder(relations, profile, aliasMap, _, child) =>
if ((profile.isInstanceOf[IncludeProfile] && profile.isEmpty) ||
!CarbonDictionaryDecoder.
@@ -130,6 +130,44 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
table.carbonTable.getTableInfo.serialize())
}
+ /**
+ * Converts to physical RDD of carbon after pushing down applicable filters.
+ * @param relation
+ * @param projects
+ * @param filterPredicates
+ * @param scanBuilder
+ * @return
+ */
+ private def pruneFilterProject(
+ relation: LogicalRelation,
+ projects: Seq[NamedExpression],
+ filterPredicates: Seq[Expression],
+ scanBuilder: (Seq[Attribute], Array[Filter],
+ ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow]) = {
+ val names = relation.catalogTable.get.partitionColumnNames
+ // Get the current partitions from table.
+ var partitions: Seq[String] = null
+ if (names.nonEmpty) {
+ val partitionSet = AttributeSet(names
+ .map(p => relation.output.find(_.name.equalsIgnoreCase(p)).get))
+ val partitionKeyFilters =
+ ExpressionSet(ExpressionSet(filterPredicates).filter(_.references.subsetOf(partitionSet)))
+ partitions =
+ CarbonFilters.getPartitions(
+ partitionKeyFilters.toSeq,
+ SparkSession.getActiveSession.get,
+ relation.catalogTable.get.identifier)
+ }
+ pruneFilterProjectRaw(
+ relation,
+ projects,
+ filterPredicates,
+ partitions,
+ (requestedColumns, _, pushedFilters, a, p) => {
+ scanBuilder(requestedColumns, pushedFilters.toArray, a, p)
+ })
+ }
+
private[this] def toCatalystRDD(
relation: LogicalRelation,
output: Seq[Attribute],
@@ -146,27 +184,13 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
}
- protected def pruneFilterProject(
- relation: LogicalRelation,
- projects: Seq[NamedExpression],
- filterPredicates: Seq[Expression],
- scanBuilder: (Seq[Attribute], Array[Filter],
- ArrayBuffer[AttributeReference]) => RDD[InternalRow]) = {
- pruneFilterProjectRaw(
- relation,
- projects,
- filterPredicates,
- (requestedColumns, _, pushedFilters, a) => {
- scanBuilder(requestedColumns, pushedFilters.toArray, a)
- })
- }
-
protected def pruneFilterProjectRaw(
relation: LogicalRelation,
rawProjects: Seq[NamedExpression],
filterPredicates: Seq[Expression],
+ partitions: Seq[String],
scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter],
- ArrayBuffer[AttributeReference]) => RDD[InternalRow]) = {
+ ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow]) = {
val projects = rawProjects.map {p =>
p.transform {
case CustomDeterministicExpression(exp) => exp
@@ -263,6 +287,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
val scan = getDataSourceScan(relation,
updateProject,
+ partitions,
scanBuilder,
candidatePredicates,
pushedFilters,
@@ -298,6 +323,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
val updateRequestedColumns = updateRequestedColumnsFunc(requestedColumns, table, needDecoder)
val scan = getDataSourceScan(relation,
updateRequestedColumns.asInstanceOf[Seq[Attribute]],
+ partitions,
scanBuilder,
candidatePredicates,
pushedFilters,
@@ -311,10 +337,11 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
}
- def getDataSourceScan(relation: LogicalRelation,
+ private def getDataSourceScan(relation: LogicalRelation,
output: Seq[Attribute],
+ partitions: Seq[String],
scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter],
- ArrayBuffer[AttributeReference]) => RDD[InternalRow],
+ ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow],
candidatePredicates: Seq[Expression],
pushedFilters: Seq[Filter],
metadata: Map[String, String],
@@ -325,14 +352,22 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
needDecoder.isEmpty) {
BatchedDataSourceScanExec(
output,
- scanBuilder(updateRequestedColumns, candidatePredicates, pushedFilters, needDecoder),
+ scanBuilder(updateRequestedColumns,
+ candidatePredicates,
+ pushedFilters,
+ needDecoder,
+ partitions),
relation.relation,
getPartitioning(table.carbonTable, updateRequestedColumns),
metadata,
relation.catalogTable.map(_.identifier), relation)
} else {
RowDataSourceScanExec(output,
- scanBuilder(updateRequestedColumns, candidatePredicates, pushedFilters, needDecoder),
+ scanBuilder(updateRequestedColumns,
+ candidatePredicates,
+ pushedFilters,
+ needDecoder,
+ partitions),
relation.relation,
getPartitioning(table.carbonTable, updateRequestedColumns),
metadata,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b8a02f39/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 7408594..a63b358 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -17,18 +17,14 @@
package org.apache.spark.sql.optimizer
+import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.CastExpressionOptimization
-import org.apache.spark.sql.CarbonBoundReference
-import org.apache.spark.sql.CastExpr
-import org.apache.spark.sql.SparkUnknownExpression
-import org.apache.spark.sql.sources
import org.apache.spark.sql.types._
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
@@ -395,4 +391,14 @@ object CarbonFilters {
case _ => expressions
}
}
+
+ def getPartitions(partitionFilters: Seq[Expression],
+ sparkSession: SparkSession,
+ identifier: TableIdentifier): Seq[String] = {
+ val partitions =
+ sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
+ partitions.toList.flatMap { partition =>
+ partition.spec.seq.map{case (column, value) => column + "=" + value}
+ }.toSet.toSeq
+ }
}
[27/50] [abbrv] carbondata git commit: [CARBONDATA-1860][PARTITION]
Support insertoverwrite for a specific partition
Posted by ja...@apache.org.
[CARBONDATA-1860][PARTITION] Support insertoverwrite for a specific partition
This closes #1700
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/47aafabb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/47aafabb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/47aafabb
Branch: refs/heads/fgdatamap
Commit: 47aafabb3bc2f64d828ed129746a5aea1bb5454c
Parents: bc2e897
Author: ravipesala <ra...@gmail.com>
Authored: Tue Dec 19 13:19:15 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Dec 21 09:06:49 2017 +0530
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 33 ++-----
.../StandardPartitionTableDropTestCase.scala | 2 +-
...tandardPartitionTableOverwriteTestCase.scala | 99 ++++++++++++++++++++
.../spark/rdd/CarbonDropPartitionRDD.scala | 5 +-
.../management/CarbonLoadDataCommand.scala | 99 ++++++++++++++++++--
.../CarbonStandardAlterTableDropPartition.scala | 24 +++--
6 files changed, 215 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index f60b69f..b44f99b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -228,7 +228,7 @@ public class PartitionMapFileStore {
}
PartitionMapper mapper = new PartitionMapper();
mapper.setPartitionMap(partitionMap);
- String path = segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT;
+ String path = segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT + ".tmp";
writePartitionFile(mapper, path);
}
}
@@ -241,33 +241,14 @@ public class PartitionMapFileStore {
* @param success
*/
public void commitPartitions(String segmentPath, final String uniqueId, boolean success) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ CarbonFile carbonFile = FileFactory
+ .getCarbonFile(segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT + ".tmp");
// write partition info to new file.
if (carbonFile.exists()) {
- CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
- }
- });
- CarbonFile latestFile = null;
- for (CarbonFile mapFile: carbonFiles) {
- if (mapFile.getName().startsWith(uniqueId)) {
- latestFile = mapFile;
- }
- }
- if (latestFile != null) {
- for (CarbonFile mapFile : carbonFiles) {
- if (latestFile != mapFile) {
- // Remove old files in case of success scenario
- if (success) {
- mapFile.delete();
- }
- }
- }
- }
- // If it is failure scenario then remove the new file.
- if (!success && latestFile != null) {
- latestFile.delete();
+ if (success) {
+ carbonFile.renameForce(segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT);
+ } else {
+ carbonFile.delete();
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
index 9a9940b..2a25255 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
@@ -156,7 +156,7 @@ class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAl
}
override def afterAll = {
- dropTable
+// dropTable
}
def dropTable = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
new file mode 100644
index 0000000..945542a
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ test("overwriting static partition table for date partition column on insert query") {
+ sql(
+ """
+ | CREATE TABLE staticpartitiondateinsert (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date,doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert overwrite table staticpartitiondateinsert PARTITION(projectenddate='2016-06-29',doj='2010-12-29 00:00:00') select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary from originTable where projectenddate=cast('2016-06-29' as Date)""")
+// sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ checkAnswer(sql("select * from staticpartitiondateinsert where projectenddate=cast('2016-06-29' as Date)"),
+ sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
+ }
+
+ test("overwriting partition table for date partition column on insert query") {
+ sql(
+ """
+ | CREATE TABLE partitiondateinsert (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date,doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)""")
+ checkAnswer(sql("select * from partitiondateinsert"),
+ sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
+ }
+
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists partitiondateinsert")
+ sql("drop table if exists staticpartitiondateinsert")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
index 09d9da1..d377c4d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -94,10 +94,11 @@ class CarbonDropPartitionRDD(
* @param tablePath
* @param segments segments to be merged
*/
-class CarbonDropPartitionRollbackRDD(
+class CarbonDropPartitionCommitRDD(
sc: SparkContext,
tablePath: String,
segments: Seq[String],
+ success: Boolean,
uniqueId: String)
extends CarbonRDD[String](sc, Nil) {
@@ -112,7 +113,7 @@ class CarbonDropPartitionRollbackRDD(
val split = theSplit.asInstanceOf[CarbonDropPartition]
logInfo("Commit partition information from : " + split.segmentPath)
- new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, false)
+ new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, success)
var havePair = false
var finished = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index a883735..69be362 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution.command.management
import java.text.SimpleDateFormat
+import java.util
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -34,7 +35,7 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.catalog.{CatalogRelation, CatalogTable}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
@@ -46,6 +47,7 @@ import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.{DictionaryServer, NonSecureDictionaryServer}
import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceProvider
@@ -53,7 +55,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.statusmanager.SegmentStatus
+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent, OperationContext, OperationListenerBus}
@@ -67,7 +69,7 @@ import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DictionaryLoadModel}
+import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD, DictionaryLoadModel}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataLoadingUtil, GlobalDictionaryUtil}
case class CarbonLoadDataCommand(
@@ -567,14 +569,18 @@ case class CarbonLoadDataCommand(
carbonLoadModel,
sparkSession)
}
- Dataset.ofRows(
- sparkSession,
+ val convertedPlan =
CarbonReflectionUtils.getInsertIntoCommand(
convertRelation,
partition,
query,
- isOverwriteTable,
- false))
+ false,
+ false)
+ if (isOverwriteTable && partition.nonEmpty) {
+ overwritePartition(sparkSession, table, convertedPlan)
+ } else {
+ Dataset.ofRows(sparkSession, convertedPlan)
+ }
}
private def convertToLogicalRelation(
@@ -597,14 +603,18 @@ case class CarbonLoadDataCommand(
}
val partitionSchema =
StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(field =>
- metastoreSchema.fields.find(_.name.equalsIgnoreCase(field.getColumnName))).map(_.get))
-
+ metastoreSchema.fields.find(_.name.equalsIgnoreCase(field.getColumnName))).map(_.get))
+ val overWriteLocal = if (overWrite && partition.nonEmpty) {
+ false
+ } else {
+ overWrite
+ }
val dataSchema =
StructType(metastoreSchema
.filterNot(field => partitionSchema.contains(field.name)))
val options = new mutable.HashMap[String, String]()
options ++= catalogTable.storage.properties
- options += (("overwrite", overWrite.toString))
+ options += (("overwrite", overWriteLocal.toString))
options += (("onepass", loadModel.getUseOnePass.toString))
options += (("dicthost", loadModel.getDictionaryServerHost))
options += (("dictport", loadModel.getDictionaryServerPort.toString))
@@ -621,6 +631,75 @@ case class CarbonLoadDataCommand(
Some(catalogTable))
}
+ /**
+ * Overwrite the partition data if static partitions are specified.
+ * @param sparkSession
+ * @param table
+ * @param logicalPlan
+ */
+ private def overwritePartition(
+ sparkSession: SparkSession,
+ table: CarbonTable,
+ logicalPlan: LogicalPlan): Unit = {
+ sparkSession.sessionState.catalog.listPartitions(
+ TableIdentifier(table.getTableName, Some(table.getDatabaseName)),
+ Some(partition.map(f => (f._1, f._2.get))))
+ val partitionNames = partition.map(k => k._1 + "=" + k._2.get).toSet
+ val uniqueId = System.currentTimeMillis().toString
+ val segments = new SegmentStatusManager(
+ table.getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments
+ try {
+ // First drop the partitions from partition mapper files of each segment
+ new CarbonDropPartitionRDD(
+ sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ partitionNames.toSeq,
+ uniqueId).collect()
+ } catch {
+ case e: Exception =>
+ // roll back the drop partitions from carbon store
+ new CarbonDropPartitionCommitRDD(
+ sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ false,
+ uniqueId).collect()
+ throw e
+ }
+
+ try {
+ Dataset.ofRows(sparkSession, logicalPlan)
+ } catch {
+ case e: Exception =>
+ // roll back the drop partitions from carbon store
+ new CarbonDropPartitionCommitRDD(
+ sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ false,
+ uniqueId).collect()
+ throw e
+ }
+ // Commit the removed partitions in carbon store.
+ new CarbonDropPartitionCommitRDD(
+ sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ true,
+ uniqueId).collect()
+ // Update the loadstatus with update time to clear cache from driver.
+ val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
+ .getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
+ CarbonUpdateUtil.updateTableMetadataStatus(
+ segmentSet,
+ table,
+ uniqueId,
+ true,
+ new util.ArrayList[String])
+ DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
+ }
+
def getDataFrameWithTupleID(): DataFrame = {
val fields = dataFrame.get.schema.fields
import org.apache.spark.sql.functions.udf
http://git-wip-us.apache.org/repos/asf/carbondata/blob/47aafabb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
index b787aa7..fd59587 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonStandardAlterTableDropPartition.scala
@@ -32,7 +32,7 @@ import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.spark.rdd.{CarbonDropPartitionRDD, CarbonDropPartitionRollbackRDD}
+import org.apache.carbondata.spark.rdd.{CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
/**
* Drop the partitions from hive and carbon store. It drops the partitions in following steps
@@ -118,20 +118,28 @@ case class CarbonStandardAlterTableDropPartition(
} catch {
case e: Exception =>
// roll back the drop partitions from carbon store
- new CarbonDropPartitionRollbackRDD(sparkSession.sparkContext,
+ new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
table.getTablePath,
segments.asScala,
+ false,
uniqueId).collect()
throw e
}
+ // commit the drop partitions from carbon store
+ new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
+ table.getTablePath,
+ segments.asScala,
+ true,
+ uniqueId).collect()
+ // Update the loadstatus with update time to clear cache from driver.
val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
.getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
- CarbonUpdateUtil
- .updateTableMetadataStatus(segmentSet,
- table,
- uniqueId,
- true,
- new util.ArrayList[String])
+ CarbonUpdateUtil.updateTableMetadataStatus(
+ segmentSet,
+ table,
+ uniqueId,
+ true,
+ new util.ArrayList[String])
DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
} finally {
AlterTableUtil.releaseLocks(locks)
[21/50] [abbrv] carbondata git commit: [CARBONDATA-1906] Update
registerTempTable method because it was marked deprecated
Posted by ja...@apache.org.
[CARBONDATA-1906] Update registerTempTable method because it was marked deprecated
Update registerTempTable method to createOrReplaceTempView, because registerTempTable was marked deprecated
This closes #1676
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/410df072
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/410df072
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/410df072
Branch: refs/heads/fgdatamap
Commit: 410df072d842f38bab7316511b0abb9752bd3461
Parents: 15f04c3
Author: xubo245 <60...@qq.com>
Authored: Tue Dec 19 15:20:51 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Dec 20 18:01:19 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/examples/CompareTest.scala | 10 +++-------
.../spark/testsuite/allqueries/AllDataTypesTestCase.scala | 2 +-
.../testsuite/allqueries/MeasureOnlyTableTestCases.scala | 2 +-
.../apache/spark/carbondata/CarbonDataSourceSuite.scala | 2 +-
4 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/410df072/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index 52ccd5f..414f8e1 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -21,8 +21,6 @@ import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
-import scala.util.Random
-
import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.types._
@@ -52,7 +50,7 @@ object CompareTest {
// +-------------+-----------+-------------+-------------+------------+
// | country | string | 1103 | dimension | yes |
// +-------------+-----------+-------------+-------------+------------+
- // | planet | string | 100,007 | dimension | yes |
+ // | planet | string | 10,007 | dimension | yes |
// +-------------+-----------+-------------+-------------+------------+
// | id | string | 10,000,000 | dimension | no |
// +-------------+-----------+-------------+-------------+------------+
@@ -67,7 +65,6 @@ object CompareTest {
// | m5 | decimal | NA | measure | no |
// +-------------+-----------+-------------+-------------+------------+
private def generateDataFrame(spark: SparkSession): DataFrame = {
- val r = new Random()
val rdd = spark.sparkContext
.parallelize(1 to 10 * 1000 * 1000, 4)
.map { x =>
@@ -257,7 +254,7 @@ object CompareTest {
.partitionBy("partitionCol")
.mode(SaveMode.Overwrite)
.parquet(table)
- spark.read.parquet(table).registerTempTable(table)
+ spark.read.parquet(table).createOrReplaceTempView(table)
}
private def loadOrcTable(spark: SparkSession, input: DataFrame, table: String): Double = time {
@@ -265,7 +262,7 @@ object CompareTest {
input.write
.mode(SaveMode.Overwrite)
.orc(table)
- spark.read.orc(table).registerTempTable(table)
+ spark.read.orc(table).createOrReplaceTempView(table)
}
private def loadCarbonTable(spark: SparkSession, input: DataFrame, tableName: String): Double = {
@@ -337,7 +334,6 @@ object CompareTest {
private def runTest(spark: SparkSession, table1: String, table2: String): Unit = {
val formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val date = new Date
- val timestamp = date.getTime
// run queries on parquet and carbon
val table1Result: Array[(Double, Array[Row])] = runQueries(spark, table1)
// do GC and sleep for some time before running next table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/410df072/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
index 699cf07..e739091 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
@@ -1085,7 +1085,7 @@ class AllDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists carbonunion")
import sqlContext.implicits._
val df = sqlContext.sparkContext.parallelize(1 to 1000).map(x => (x+"", (x+100)+"")).toDF("c1", "c2")
- df.registerTempTable("sparkunion")
+ df.createOrReplaceTempView("sparkunion")
df.write
.format("carbondata")
.mode(SaveMode.Overwrite)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/410df072/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
index 0a3869c..ab75c0a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/MeasureOnlyTableTestCases.scala
@@ -376,7 +376,7 @@ class MeasureOnlyTableTestCases extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists carbonunion")
import sqlContext.implicits._
val df = sqlContext.sparkContext.parallelize(1 to 1000).map(x => (x, (x+100))).toDF("c1", "c2")
- df.registerTempTable("sparkunion")
+ df.createOrReplaceTempView("sparkunion")
df.write
.format("carbondata")
.mode(SaveMode.Overwrite)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/410df072/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index cf465fb..0f934cb 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -147,7 +147,7 @@ class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll {
sql("drop table if exists sparkunion")
import sqlContext.implicits._
val df = sqlContext.sparkContext.parallelize(1 to 1000).map(x => (x+"", (x+100)+"")).toDF("c1", "c2")
- df.registerTempTable("sparkunion")
+ df.createOrReplaceTempView("sparkunion")
df.write
.format("carbondata")
.mode(SaveMode.Overwrite)
[34/50] [abbrv] carbondata git commit: [CARBONDATA-1863][PARTITION]
Supported clean files for partition table.
Posted by ja...@apache.org.
[CARBONDATA-1863][PARTITION] Supported clean files for partition table.
This closes #1706
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9659edcc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9659edcc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9659edcc
Branch: refs/heads/fgdatamap
Commit: 9659edccbd05e9f5499911e6d049d9b9e1cf8c3a
Parents: 577a8b0
Author: ravipesala <ra...@gmail.com>
Authored: Thu Dec 21 10:23:27 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Dec 21 21:50:42 2017 +0530
----------------------------------------------------------------------
.../blockletindex/SegmentIndexFileStore.java | 2 +-
.../core/metadata/PartitionMapFileStore.java | 147 ++++++++++++++--
.../core/writer/CarbonIndexFileMergeWriter.java | 41 +++--
.../StandardPartitionTableCleanTestCase.scala | 167 +++++++++++++++++++
.../StandardPartitionTableDropTestCase.scala | 2 +-
.../org/apache/carbondata/api/CarbonStore.scala | 10 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 29 ++--
.../management/CarbonCleanFilesCommand.scala | 15 +-
.../management/CarbonLoadDataCommand.scala | 37 ++--
.../datasources/CarbonFileFormat.scala | 9 +-
.../loading/model/CarbonLoadModel.java | 6 +-
.../processing/util/CarbonLoaderUtil.java | 3 +
12 files changed, 414 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index 244e8bb..01cb1d7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -42,7 +42,7 @@ public class SegmentIndexFileStore {
*/
private Map<String, byte[]> carbonIndexMap;
- public SegmentIndexFileStore() throws IOException {
+ public SegmentIndexFileStore() {
carbonIndexMap = new HashMap<>();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index b44f99b..d29dfbb 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -38,8 +39,18 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import com.google.gson.Gson;
@@ -134,6 +145,31 @@ public class PartitionMapFileStore {
}
}
+ private String getPartitionFilePath(String segmentPath) {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ if (carbonFile.exists()) {
+ CarbonFile[] partitionFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
+ }
+ });
+ if (partitionFiles != null && partitionFiles.length > 0) {
+ partionedSegment = true;
+ int i = 0;
+ // Get the latest partition map file based on the timestamp of that file.
+ long[] partitionTimestamps = new long[partitionFiles.length];
+ for (CarbonFile file : partitionFiles) {
+ partitionTimestamps[i++] = Long.parseLong(file.getName()
+ .substring(0, file.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
+ }
+ Arrays.sort(partitionTimestamps);
+ return segmentPath + "/" + partitionTimestamps[partitionTimestamps.length - 1]
+ + CarbonTablePath.PARTITION_MAP_EXT;
+ }
+ }
+ return null;
+ }
+
private CarbonFile[] getPartitionFiles(String segmentPath) {
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
if (carbonFile.exists()) {
@@ -179,22 +215,15 @@ public class PartitionMapFileStore {
return partitionMapper;
}
+ /**
+ * Reads all partitions which existed inside the passed segment path
+ * @param segmentPath
+ */
public void readAllPartitionsOfSegment(String segmentPath) {
- CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
- if (partitionFiles != null && partitionFiles.length > 0) {
+ String partitionFilePath = getPartitionFilePath(segmentPath);
+ if (partitionFilePath != null) {
partionedSegment = true;
- int i = 0;
- // Get the latest partition map file based on the timestamp of that file.
- long [] partitionTimestamps = new long[partitionFiles.length];
- for (CarbonFile file : partitionFiles) {
- partitionTimestamps[i++] =
- Long.parseLong(file.getName().substring(
- 0, file.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
- }
- Arrays.sort(partitionTimestamps);
- PartitionMapper partitionMapper = readPartitionMap(
- segmentPath + "/" + partitionTimestamps[partitionTimestamps.length - 1]
- + CarbonTablePath.PARTITION_MAP_EXT);
+ PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
partitionMap.putAll(partitionMapper.getPartitionMap());
}
}
@@ -253,6 +282,96 @@ public class PartitionMapFileStore {
}
}
+ /**
+ * Clean up invalid data after drop partition in all segments of table
+ * @param table
+ * @param currentPartitions Current partitions of table
+ * @param forceDelete Whether it should be deleted force or check the time for an hour creation
+ * to delete data.
+ * @throws IOException
+ */
+ public void cleanSegments(
+ CarbonTable table,
+ List<String> currentPartitions,
+ boolean forceDelete) throws IOException {
+ SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
+
+ CarbonTablePath carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(),
+ table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
+
+ LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
+ // scan through each segment.
+
+ for (LoadMetadataDetails segment : details) {
+
+ // if this segment is valid then only we will go for deletion of related
+ // dropped partition files. if the segment is mark for delete or compacted then any way
+ // it will get deleted.
+
+ if (segment.getSegmentStatus() == SegmentStatus.SUCCESS
+ || segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
+ List<String> toBeDeletedIndexFiles = new ArrayList<>();
+ List<String> toBeDeletedDataFiles = new ArrayList<>();
+ // take the list of files from this segment.
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
+ String partitionFilePath = getPartitionFilePath(segmentPath);
+ if (partitionFilePath != null) {
+ PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
+ indexFileStore.readAllIIndexOfSegment(segmentPath);
+ Set<String> indexFilesFromSegment = indexFileStore.getCarbonIndexMap().keySet();
+ for (String indexFile : indexFilesFromSegment) {
+ // Check the partition information in the partiton mapper
+ List<String> indexPartitions = partitionMapper.partitionMap.get(indexFile);
+ if (indexPartitions == null || !currentPartitions.containsAll(indexPartitions)) {
+ Long fileTimestamp = CarbonUpdateUtil.getTimeStampAsLong(indexFile
+ .substring(indexFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
+ indexFile.length() - CarbonTablePath.INDEX_FILE_EXT.length()));
+ if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
+ toBeDeletedIndexFiles.add(indexFile);
+ // Add the corresponding carbondata files to the delete list.
+ byte[] fileData = indexFileStore.getFileData(indexFile);
+ List<DataFileFooter> indexInfo =
+ fileFooterConverter.getIndexInfo(segmentPath + "/" + indexFile, fileData);
+ for (DataFileFooter footer : indexInfo) {
+ toBeDeletedDataFiles.add(footer.getBlockInfo().getTableBlockInfo().getFilePath());
+ }
+ }
+ }
+ }
+
+ if (toBeDeletedIndexFiles.size() > 0) {
+ indexFilesFromSegment.removeAll(toBeDeletedIndexFiles);
+ new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath,
+ new ArrayList<String>(indexFilesFromSegment));
+ for (String dataFile : toBeDeletedDataFiles) {
+ FileFactory.deleteFile(dataFile, FileFactory.getFileType(dataFile));
+ }
+ }
+ CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
+ CarbonFile currentPartitionFile = FileFactory.getCarbonFile(partitionFilePath);
+ if (partitionFiles != null) {
+ // Delete all old partition files
+ for (CarbonFile partitionFile : partitionFiles) {
+ if (!currentPartitionFile.getName().equalsIgnoreCase(partitionFile.getName())) {
+ partitionFile.delete();
+ }
+ }
+ }
+ partitionMapper = readPartitionMap(partitionFilePath);
+ if (partitionMapper != null) {
+ // delete partition map if there is no partition files exist
+ if (partitionMapper.partitionMap.size() == 0) {
+ currentPartitionFile.delete();
+ }
+ }
+ }
+ }
+ }
+ }
+
public List<String> getPartitions(String indexFileName) {
return partitionMap.get(indexFileName);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index 85f08cc..067d024 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -39,36 +39,53 @@ public class CarbonIndexFileMergeWriter {
/**
* Merge all the carbonindex files of segment to a merged file
* @param segmentPath
+ * @param indexFileNamesTobeAdded while merging it comsiders only these files.
+ * If null then consider all
* @throws IOException
*/
- public void mergeCarbonIndexFilesOfSegment(String segmentPath) throws IOException {
+ public void mergeCarbonIndexFilesOfSegment(
+ String segmentPath,
+ List<String> indexFileNamesTobeAdded) throws IOException {
CarbonFile[] indexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath);
- if (isCarbonIndexFilePresent(indexFiles)) {
+ if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
fileStore.readAllIIndexOfSegment(segmentPath);
- openThriftWriter(
- segmentPath + "/" +
- System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT);
Map<String, byte[]> indexMap = fileStore.getCarbonIndexMap();
MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();
MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();
List<String> fileNames = new ArrayList<>(indexMap.size());
List<ByteBuffer> data = new ArrayList<>(indexMap.size());
for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) {
- fileNames.add(entry.getKey());
- data.add(ByteBuffer.wrap(entry.getValue()));
+ if (indexFileNamesTobeAdded == null ||
+ indexFileNamesTobeAdded.contains(entry.getKey())) {
+ fileNames.add(entry.getKey());
+ data.add(ByteBuffer.wrap(entry.getValue()));
+ }
+ }
+ if (fileNames.size() > 0) {
+ openThriftWriter(
+ segmentPath + "/" + System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT);
+ indexHeader.setFile_names(fileNames);
+ mergedBlockIndex.setFileData(data);
+ writeMergedBlockIndexHeader(indexHeader);
+ writeMergedBlockIndex(mergedBlockIndex);
+ close();
}
- indexHeader.setFile_names(fileNames);
- mergedBlockIndex.setFileData(data);
- writeMergedBlockIndexHeader(indexHeader);
- writeMergedBlockIndex(mergedBlockIndex);
- close();
for (CarbonFile indexFile : indexFiles) {
indexFile.delete();
}
}
}
+ /**
+ * Merge all the carbonindex files of segment to a merged file
+ * @param segmentPath
+ * @throws IOException
+ */
+ public void mergeCarbonIndexFilesOfSegment(String segmentPath) throws IOException {
+ mergeCarbonIndexFilesOfSegment(segmentPath, null);
+ }
+
private boolean isCarbonIndexFilePresent(CarbonFile[] indexFiles) {
for (CarbonFile file : indexFiles) {
if (file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
new file mode 100644
index 0000000..2b0dd09
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int, partitionMapFiles: Int): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
+ carbonTable.getTablePath)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".carbondata")
+ }
+ })
+ assert(dataFiles.length == partitions)
+ val partitionFile = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".partitionmap")
+ }
+ })
+ assert(partitionFile.length == partitionMapFiles)
+ }
+
+ test("clean up partition table for int partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(
+ sql(s"""select count (*) from partitionone"""),
+ sql(s"""select count (*) from originTable"""))
+
+ checkAnswer(
+ sql(s"""select count (*) from partitionone where empno=11"""),
+ sql(s"""select count (*) from originTable where empno=11"""))
+
+ sql(s"""ALTER TABLE partitionone DROP PARTITION(empno='11')""")
+ validateDataFiles("default_partitionone", "0", 10, 2)
+ sql(s"CLEAN FILES FOR TABLE partitionone").show()
+
+ checkExistence(sql(s"""SHOW PARTITIONS partitionone"""), false, "empno=11")
+ validateDataFiles("default_partitionone", "0", 9, 1)
+ checkAnswer(
+ sql(s"""select count (*) from partitionone where empno=11"""),
+ Seq(Row(0)))
+
+ }
+
+ test("clean up partition on table for more partition columns") {
+ sql(
+ """
+ | CREATE TABLE partitionmany (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""ALTER TABLE partitionmany DROP PARTITION(deptname='Learning')""")
+ validateDataFiles("default_partitionmany", "0", 10, 2)
+ validateDataFiles("default_partitionmany", "1", 10, 2)
+ sql(s"CLEAN FILES FOR TABLE partitionmany").show()
+ validateDataFiles("default_partitionmany", "0", 8, 1)
+ validateDataFiles("default_partitionmany", "1", 8, 1)
+ checkExistence(sql(s"""SHOW PARTITIONS partitionmany"""), false, "deptname=Learning", "projectcode=928479")
+ checkAnswer(
+ sql(s"""select count (*) from partitionmany where deptname='Learning'"""),
+ Seq(Row(0)))
+ }
+
+ test("clean up after dropping all partition on table") {
+ sql(
+ """
+ | CREATE TABLE partitionall (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionall OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionall OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='Learning')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='configManagement')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='network')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='protocol')""")
+ sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='security')""")
+ assert(sql(s"""SHOW PARTITIONS partitionall""").collect().length == 0)
+ validateDataFiles("default_partitionall", "0", 10, 6)
+ sql(s"CLEAN FILES FOR TABLE partitionall").show()
+ validateDataFiles("default_partitionall", "0", 0, 0)
+ checkAnswer(
+ sql(s"""select count (*) from partitionall"""),
+ Seq(Row(0)))
+ }
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitionall")
+ sql("drop table if exists partitionmany")
+ sql("drop table if exists partitionshow")
+ sql("drop table if exists staticpartition")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
index 2a25255..9a9940b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
@@ -156,7 +156,7 @@ class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAl
}
override def afterAll = {
-// dropTable
+ dropTable
}
def dropTable = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 2b127e4..d514f77 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -30,7 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, PartitionMapFileStore}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -103,7 +103,8 @@ object CarbonStore {
tableName: String,
storePath: String,
carbonTable: CarbonTable,
- forceTableClean: Boolean): Unit = {
+ forceTableClean: Boolean,
+ currentTablePartitions: Option[Seq[String]] = None): Unit = {
LOGGER.audit(s"The clean files request has been received for $dbName.$tableName")
var carbonCleanFilesLock: ICarbonLock = null
var absoluteTableIdentifier: AbsoluteTableIdentifier = null
@@ -128,6 +129,11 @@ object CarbonStore {
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = true, carbonTable)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
+ currentTablePartitions match {
+ case Some(partitions) =>
+ new PartitionMapFileStore().cleanSegments(carbonTable, partitions.asJava, true)
+ case _ =>
+ }
}
} finally {
if (carbonCleanFilesLock != null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index f78412b..4c405a4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -157,16 +157,25 @@ object CarbonScalaUtil {
def convertToUTF8String(value: String,
dataType: DataType,
timeStampFormat: SimpleDateFormat,
- dateFormat: SimpleDateFormat): UTF8String = {
- dataType match {
- case TimestampType =>
- UTF8String.fromString(
- DateTimeUtils.timestampToString(timeStampFormat.parse(value).getTime * 1000))
- case DateType =>
- UTF8String.fromString(
- DateTimeUtils.dateToString(
- (dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt))
- case _ => UTF8String.fromString(value)
+ dateFormat: SimpleDateFormat,
+ serializationNullFormat: String): UTF8String = {
+ if (value == null || serializationNullFormat.equals(value)) {
+ return UTF8String.fromString(value)
+ }
+ try {
+ dataType match {
+ case TimestampType =>
+ UTF8String.fromString(
+ DateTimeUtils.timestampToString(timeStampFormat.parse(value).getTime * 1000))
+ case DateType =>
+ UTF8String.fromString(
+ DateTimeUtils.dateToString(
+ (dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt))
+ case _ => UTF8String.fromString(value)
+ }
+ } catch {
+ case e: Exception =>
+ UTF8String.fromString(value)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index e0530f6..342acd4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -18,7 +18,10 @@
package org.apache.spark.sql.execution.command.management
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.execution.command.{Checker, DataCommand}
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.core.util.CarbonProperties
@@ -76,13 +79,21 @@ case class CarbonCleanFilesCommand(
private def cleanGarbageData(sparkSession: SparkSession,
databaseNameOp: Option[String], tableName: String): Unit = {
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
-
+ val partitions: Option[Seq[String]] = if (carbonTable.isHivePartitionTable) {
+ Some(CarbonFilters.getPartitions(
+ Seq.empty[Expression],
+ sparkSession,
+ TableIdentifier(tableName, databaseNameOp)))
+ } else {
+ None
+ }
CarbonStore.cleanFiles(
CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
tableName,
CarbonProperties.getStorePath,
carbonTable,
- forceTableClean)
+ forceTableClean,
+ partitions)
}
private def cleanGarbageDataInAllTables(sparkSession: SparkSession): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index f96c0a7..7285d9d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -34,14 +34,15 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedAttribute}
-import org.apache.spark.sql.catalyst.catalog.{CatalogRelation, CatalogTable}
-import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY
import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.types.{StringType, StructField, StructType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
@@ -52,6 +53,7 @@ import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.{DictionaryServer, NonSecureDictionaryServer}
import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceProvider
+import org.apache.carbondata.core.metadata.PartitionMapFileStore
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
@@ -453,16 +455,28 @@ case class CarbonLoadDataCommand(
hadoopConf: Configuration,
dataFrame: Option[DataFrame]) = {
val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val identifier = TableIdentifier(table.getTableName, Some(table.getDatabaseName))
val logicalPlan =
sparkSession.sessionState.catalog.lookupRelation(
- TableIdentifier(table.getTableName, Some(table.getDatabaseName)))
+ identifier)
val relation = logicalPlan.collect {
case l: LogicalRelation => l
- case c: CatalogRelation => c
+ case c // To make compatabile with spark 2.1 and 2.2 we need to compare classes
+ if (c.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
+ c.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
+ c.getClass.getName.equals(
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation")) => c
}.head
-
+ // Clean up the old invalid segment data.
+ DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, table)
+ val currentPartitions =
+ CarbonFilters.getPartitions(Seq.empty[Expression], sparkSession, identifier)
+ // Clean up the alreday dropped partitioned data
+ new PartitionMapFileStore().cleanSegments(table, currentPartitions.asJava, false)
// Converts the data to carbon understandable format. The timestamp/date format data needs to
// converted to hive standard fomat to let spark understand the data to partition.
+ val serializationNullFormat =
+ carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
val query: LogicalPlan = if (dataFrame.isDefined) {
var timeStampformatString = CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
val timeStampFormat = new SimpleDateFormat(timeStampformatString)
@@ -545,13 +559,15 @@ case class CarbonLoadDataCommand(
jobConf).map{ case (key, value) =>
val data = new Array[Any](len)
var i = 0
- while (i < len) {
+ val input = value.get()
+ while (i < input.length) {
// TODO find a way to avoid double conversion of date and time.
data(i) = CarbonScalaUtil.convertToUTF8String(
- value.get()(i),
+ input(i),
rowDataTypes(i),
timeStampFormat,
- dateFormat)
+ dateFormat,
+ serializationNullFormat)
i = i + 1
}
InternalRow.fromSeq(data)
@@ -576,7 +592,7 @@ case class CarbonLoadDataCommand(
isOverwriteTable,
carbonLoadModel,
sparkSession)
- case c: CatalogRelation =>
+ case others =>
val catalogTable = CarbonReflectionUtils.getFieldOfCatalogTable(
"tableMeta",
relation).asInstanceOf[CatalogTable]
@@ -587,7 +603,7 @@ case class CarbonLoadDataCommand(
val catalog = new CatalogFileIndex(sparkSession, catalogTable, sizeInBytes)
convertToLogicalRelation(
catalogTable,
- c.output,
+ others.output,
sizeInBytes,
isOverwriteTable,
carbonLoadModel,
@@ -642,6 +658,7 @@ case class CarbonLoadDataCommand(
options += (("onepass", loadModel.getUseOnePass.toString))
options += (("dicthost", loadModel.getDictionaryServerHost))
options += (("dictport", loadModel.getDictionaryServerPort.toString))
+ options ++= this.options
if (updateModel.isDefined) {
options += (("updatetimestamp", updateModel.get.updatedTimeStamp.toString))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index a95693c..b9df7a1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -189,7 +189,14 @@ private class CarbonOutputWriter(path: String,
extends OutputWriter with AbstractCarbonOutputWriter {
val partitions = getPartitionsFromPath(path, context).map(ExternalCatalogUtils.unescapePathName)
val partitionData = if (partitions.nonEmpty) {
- partitions.map(_.split("=")(1))
+ partitions.map{ p =>
+ val splitData = p.split("=")
+ if (splitData.length > 1) {
+ splitData(1)
+ } else {
+ ""
+ }
+ }
} else {
Array.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 3031b8e..8a295d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -564,7 +564,11 @@ public class CarbonLoadModel implements Serializable {
* @return
*/
public LoadMetadataDetails getCurrentLoadMetadataDetail() {
- return loadMetadataDetails.get(loadMetadataDetails.size() - 1);
+ if (loadMetadataDetails != null && loadMetadataDetails.size() > 0) {
+ return loadMetadataDetails.get(loadMetadataDetails.size() - 1);
+ } else {
+ return null;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9659edcc/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index fef6930..49ca254 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -393,6 +393,9 @@ public final class CarbonLoaderUtil {
SegmentStatus loadStatus = SegmentStatus.MARKED_FOR_DELETE;
// always the last entry in the load metadata details will be the current load entry
LoadMetadataDetails loadMetaEntry = model.getCurrentLoadMetadataDetail();
+ if (loadMetaEntry == null) {
+ return;
+ }
CarbonLoaderUtil
.populateNewLoadMetaEntry(loadMetaEntry, loadStatus, model.getFactTimeStamp(), true);
boolean entryAdded =
[47/50] [abbrv] carbondata git commit: [CARBONDATA-1911] Added Insert
into query test case for parquet & hive from Carbon Table
Posted by ja...@apache.org.
[CARBONDATA-1911] Added Insert into query test case for parquet & hive from Carbon Table
Added test cases for insert into operation for parquet & hive from Carbon Table
This closes #1683
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a295b5f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a295b5f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a295b5f
Branch: refs/heads/fgdatamap
Commit: 8a295b5ffd9a00d2997c67cf1e3f3e73d015ad38
Parents: 7997f22
Author: Manohar <ma...@gmail.com>
Authored: Tue Dec 19 18:47:16 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Dec 23 21:34:02 2017 +0800
----------------------------------------------------------------------
.../InsertIntoNonCarbonTableTestCase.scala | 108 +++++++++++++++++++
1 file changed, 108 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a295b5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/insertQuery/InsertIntoNonCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/insertQuery/InsertIntoNonCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/insertQuery/InsertIntoNonCarbonTableTestCase.scala
new file mode 100644
index 0000000..a745672
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/insertQuery/InsertIntoNonCarbonTableTestCase.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.testsuite.insertQuery
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+
+class InsertIntoNonCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll {
+ sql("drop table if exists TCarbonSource")
+ sql(
+ "create table TCarbonSource (imei string,deviceInformationId int,MAC string,deviceColor " +
+ "string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize " +
+ "string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode " +
+ "string,internalModels string, deliveryTime string, channelsId string, channelsName string " +
+ ", deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity " +
+ "string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, " +
+ "ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, " +
+ "Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, " +
+ "Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, " +
+ "Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, " +
+ "Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion" +
+ " string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, " +
+ "Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country " +
+ "string, Latest_province string, Latest_city string, Latest_district string, Latest_street " +
+ "string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string," +
+ " Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, " +
+ "Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, " +
+ "Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, " +
+ "Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber " +
+ "BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"LOAD DATA INPATH '$resourcesPath/100_olap.csv' INTO table TCarbonSource options " +
+ "('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor," +
+ "device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series," +
+ "productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId," +
+ "deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet," +
+ "oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity," +
+ "ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion," +
+ "Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion," +
+ "Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion," +
+ "Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR," +
+ "Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street," +
+ "Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber," +
+ "Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer," +
+ "Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions," +
+ "Latest_operatorId,gamePointDescription,gamePointId,contractNumber', " +
+ "'bad_records_logger_enable'='false','bad_records_action'='FORCE')")
+ }
+
+ test("insert into hive") {
+ sql("drop table if exists thive2")
+ sql(
+ "create table thive2 row format delimited fields terminated by '\017' stored as textfile as" +
+ " select imei,deviceInformationId,MAC from TCarbonSource")
+ checkAnswer(
+ sql(
+ "select imei,deviceInformationId,MAC from TCarbonSource order by imei, " +
+ "deviceInformationId,MAC"),
+ sql("select imei,deviceInformationId,MAC from thive2 order by imei,deviceInformationId,MAC")
+ )
+ sql("drop table thive2")
+ }
+ test("insert into parquet") {
+ sql("drop table if exists tparquet")
+ sql("create table tparquet(imei string,deviceInformationId int) STORED AS PARQUET")
+ sql("insert into tparquet select imei,deviceInformationId from TCarbonSource")
+ checkAnswer(
+ sql("select imei,deviceInformationId from TCarbonSource order by imei,deviceInformationId"),
+ sql("select imei,deviceInformationId from tparquet order by imei,deviceInformationId")
+ )
+ sql("drop table tparquet")
+ }
+ test("insert into hive conditional") {
+ sql("drop table if exists thive_cond")
+ sql(
+ "create table thive_cond row format delimited fields terminated by '\017' stored as " +
+ "textfile as SELECT(CASE WHEN imei IS NOT NULL THEN imei ELSE MAC END) AS temp FROM " +
+ "TCarbonSource limit 10")
+ checkAnswer(
+ sql("select count(*) from thive_cond"),
+ Seq(Row(10))
+ )
+ sql("drop table thive_cond")
+ }
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS TCarbonSource")
+ }
+}
\ No newline at end of file
[19/50] [abbrv] carbondata git commit: [CARBONDATA-1907] Avoid
unnecessary logging to improve query performance for no dictionary non string
columns
Posted by ja...@apache.org.
[CARBONDATA-1907] Avoid unnecessary logging to improve query performance for no dictionary non string columns
Changes done to return null in case of no dictionary column for non string data types when data is empty.This is done to avoid excessive logging which is impacting the query performance.
This closes #1679
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a86c54b3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a86c54b3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a86c54b3
Branch: refs/heads/fgdatamap
Commit: a86c54b3e83fef0b45ed70c50ee7483f5cc0894c
Parents: 694ee77
Author: manishgupta88 <to...@gmail.com>
Authored: Tue Dec 19 17:27:30 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Wed Dec 20 14:06:09 2017 +0530
----------------------------------------------------------------------
.../executor/impl/AbstractQueryExecutor.java | 11 ++++-----
.../carbondata/core/util/DataTypeUtil.java | 24 ++++++++++++++++++++
.../carbondata/core/util/DataTypeUtilTest.java | 18 +++++++++++++++
3 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 88a8238..cb193e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
@@ -544,12 +543,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
}
UnsafeMemoryManager.INSTANCE.freeMemoryAll(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId());
if (null != queryProperties.executorService) {
- queryProperties.executorService.shutdown();
- try {
- queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new QueryExecutionException(e);
- }
+ // In case of limit query when number of limit records is already found so executors
+ // must stop all the running execution otherwise it will keep running and will hit
+ // the query performance.
+ queryProperties.executorService.shutdownNow();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8c13a3..0e819c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -353,12 +353,26 @@ public final class DataTypeUtil {
} else if (actualDataType == DataTypes.STRING) {
return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
} else if (actualDataType == DataTypes.SHORT) {
+ // for non string type no dictionary column empty byte array is empty value
+ // so no need to parse
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.INT) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.LONG) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
} else if (actualDataType == DataTypes.TIMESTAMP) {
+ if (isEmptyByteArray(dataInBytes)) {
+ return null;
+ }
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length) * 1000L;
} else {
return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
@@ -372,6 +386,16 @@ public final class DataTypeUtil {
}
}
+ /**
+ * Method to check if byte array is empty
+ *
+ * @param dataInBytes
+ * @return
+ */
+ private static boolean isEmptyByteArray(byte[] dataInBytes) {
+ return dataInBytes.length == 0;
+ }
+
/**
* Below method will be used to convert the data passed to its actual data
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index c944d18..bb528b9 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -98,6 +98,24 @@ public class DataTypeUtilTest {
}
+ @Test public void testGetDataBasedOnDataTypeForNoDictionaryColumn() {
+ Object result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.INT);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.SHORT);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.LONG);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.TIMESTAMP);
+ assert (result == null);
+ result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+ DataTypes.STRING);
+ assert (result != null);
+ }
+
}